i915_gem_pages.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2014-2016 Intel Corporation
  4 */
  5
  6#include <linux/vmalloc.h>
  7
  8#include <drm/drm_cache.h>
  9#include <drm/drm_panic.h>
 10#include <drm/drm_print.h>
 11
 12#include "display/intel_fb.h"
 13#include "display/intel_display_types.h"
 14#include "gt/intel_gt.h"
 15#include "gt/intel_tlb.h"
 16
 17#include "i915_drv.h"
 18#include "i915_gem_object.h"
 19#include "i915_scatterlist.h"
 20#include "i915_gem_lmem.h"
 21#include "i915_gem_mman.h"
 22
 23void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
 24				 struct sg_table *pages)
 25{
 26	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 27	unsigned long supported = RUNTIME_INFO(i915)->page_sizes;
 28	bool shrinkable;
 29	int i;
 30
 31	assert_object_held_shared(obj);
 32
 33	if (i915_gem_object_is_volatile(obj))
 34		obj->mm.madv = I915_MADV_DONTNEED;
 35
 36	/* Make the pages coherent with the GPU (flushing any swapin). */
 37	if (obj->cache_dirty) {
 38		WARN_ON_ONCE(IS_DGFX(i915));
 39		obj->write_domain = 0;
 40		if (i915_gem_object_has_struct_page(obj))
 41			drm_clflush_sg(pages);
 42		obj->cache_dirty = false;
 43	}
 44
 45	obj->mm.get_page.sg_pos = pages->sgl;
 46	obj->mm.get_page.sg_idx = 0;
 47	obj->mm.get_dma_page.sg_pos = pages->sgl;
 48	obj->mm.get_dma_page.sg_idx = 0;
 49
 50	obj->mm.pages = pages;
 51
 52	obj->mm.page_sizes.phys = i915_sg_dma_sizes(pages->sgl);
 53	GEM_BUG_ON(!obj->mm.page_sizes.phys);
 54
 55	/*
 56	 * Calculate the supported page-sizes which fit into the given
 57	 * sg_page_sizes. This will give us the page-sizes which we may be able
 58	 * to use opportunistically when later inserting into the GTT. For
 59	 * example if phys=2G, then in theory we should be able to use 1G, 2M,
 60	 * 64K or 4K pages, although in practice this will depend on a number of
 61	 * other factors.
 62	 */
 63	obj->mm.page_sizes.sg = 0;
 64	for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
 65		if (obj->mm.page_sizes.phys & ~0u << i)
 66			obj->mm.page_sizes.sg |= BIT(i);
 67	}
 68	GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
 69
 70	shrinkable = i915_gem_object_is_shrinkable(obj);
 71
 72	if (i915_gem_object_is_tiled(obj) &&
 73	    i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) {
 74		GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
 75		i915_gem_object_set_tiling_quirk(obj);
 76		GEM_BUG_ON(!list_empty(&obj->mm.link));
 77		atomic_inc(&obj->mm.shrink_pin);
 78		shrinkable = false;
 79	}
 80
 81	if (shrinkable && !i915_gem_object_has_self_managed_shrink_list(obj)) {
 82		struct list_head *list;
 83		unsigned long flags;
 84
 85		assert_object_held(obj);
 86		spin_lock_irqsave(&i915->mm.obj_lock, flags);
 87
 88		i915->mm.shrink_count++;
 89		i915->mm.shrink_memory += obj->base.size;
 90
 91		if (obj->mm.madv != I915_MADV_WILLNEED)
 92			list = &i915->mm.purge_list;
 93		else
 94			list = &i915->mm.shrink_list;
 95		list_add_tail(&obj->mm.link, list);
 96
 97		atomic_set(&obj->mm.shrink_pin, 0);
 98		spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
 99	}
100}
101
102int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
103{
104	struct drm_i915_private *i915 = to_i915(obj->base.dev);
105	int err;
106
107	assert_object_held_shared(obj);
108
109	if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
110		drm_dbg(&i915->drm,
111			"Attempting to obtain a purgeable object\n");
112		return -EFAULT;
113	}
114
115	err = obj->ops->get_pages(obj);
116	GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj));
117
118	return err;
119}
120
121/* Ensure that the associated pages are gathered from the backing storage
122 * and pinned into our object. i915_gem_object_pin_pages() may be called
123 * multiple times before they are released by a single call to
124 * i915_gem_object_unpin_pages() - once the pages are no longer referenced
125 * either as a result of memory pressure (reaping pages under the shrinker)
126 * or as the object is itself released.
127 */
128int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
129{
130	int err;
131
132	assert_object_held(obj);
133
134	assert_object_held_shared(obj);
135
136	if (unlikely(!i915_gem_object_has_pages(obj))) {
137		GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
138
139		err = ____i915_gem_object_get_pages(obj);
140		if (err)
141			return err;
142
143		smp_mb__before_atomic();
144	}
145	atomic_inc(&obj->mm.pages_pin_count);
146
147	return 0;
148}
149
150int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj)
151{
152	struct i915_gem_ww_ctx ww;
153	int err;
154
155	i915_gem_ww_ctx_init(&ww, true);
156retry:
157	err = i915_gem_object_lock(obj, &ww);
158	if (!err)
159		err = i915_gem_object_pin_pages(obj);
160
161	if (err == -EDEADLK) {
162		err = i915_gem_ww_ctx_backoff(&ww);
163		if (!err)
164			goto retry;
165	}
166	i915_gem_ww_ctx_fini(&ww);
167	return err;
168}
169
170/* Immediately discard the backing storage */
171int i915_gem_object_truncate(struct drm_i915_gem_object *obj)
172{
173	if (obj->ops->truncate)
174		return obj->ops->truncate(obj);
175
176	return 0;
177}
178
179static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
180{
181	struct radix_tree_iter iter;
182	void __rcu **slot;
183
184	rcu_read_lock();
185	radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
186		radix_tree_delete(&obj->mm.get_page.radix, iter.index);
187	radix_tree_for_each_slot(slot, &obj->mm.get_dma_page.radix, &iter, 0)
188		radix_tree_delete(&obj->mm.get_dma_page.radix, iter.index);
189	rcu_read_unlock();
190}
191
192static void unmap_object(struct drm_i915_gem_object *obj, void *ptr)
193{
194	if (is_vmalloc_addr(ptr))
195		vunmap(ptr);
196}
197
198static void flush_tlb_invalidate(struct drm_i915_gem_object *obj)
199{
200	struct drm_i915_private *i915 = to_i915(obj->base.dev);
201	struct intel_gt *gt;
202	int id;
203
204	for_each_gt(gt, i915, id) {
205		if (!obj->mm.tlb[id])
206			continue;
207
208		intel_gt_invalidate_tlb_full(gt, obj->mm.tlb[id]);
209		obj->mm.tlb[id] = 0;
210	}
211}
212
213struct sg_table *
214__i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
215{
216	struct sg_table *pages;
217
218	assert_object_held_shared(obj);
219
220	pages = fetch_and_zero(&obj->mm.pages);
221	if (IS_ERR_OR_NULL(pages))
222		return pages;
223
224	if (i915_gem_object_is_volatile(obj))
225		obj->mm.madv = I915_MADV_WILLNEED;
226
227	if (!i915_gem_object_has_self_managed_shrink_list(obj))
228		i915_gem_object_make_unshrinkable(obj);
229
230	if (obj->mm.mapping) {
231		unmap_object(obj, page_mask_bits(obj->mm.mapping));
232		obj->mm.mapping = NULL;
233	}
234
235	__i915_gem_object_reset_page_iter(obj);
236	obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
237
238	flush_tlb_invalidate(obj);
239
240	return pages;
241}
242
243int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
244{
245	struct sg_table *pages;
246
247	if (i915_gem_object_has_pinned_pages(obj))
248		return -EBUSY;
249
250	/* May be called by shrinker from within get_pages() (on another bo) */
251	assert_object_held_shared(obj);
252
253	i915_gem_object_release_mmap_offset(obj);
254
255	/*
256	 * ->put_pages might need to allocate memory for the bit17 swizzle
257	 * array, hence protect them from being reaped by removing them from gtt
258	 * lists early.
259	 */
260	pages = __i915_gem_object_unset_pages(obj);
261
262	/*
263	 * XXX Temporary hijinx to avoid updating all backends to handle
264	 * NULL pages. In the future, when we have more asynchronous
265	 * get_pages backends we should be better able to handle the
266	 * cancellation of the async task in a more uniform manner.
267	 */
268	if (!IS_ERR_OR_NULL(pages))
269		obj->ops->put_pages(obj, pages);
270
271	return 0;
272}
273
274/* The 'mapping' part of i915_gem_object_pin_map() below */
275static void *i915_gem_object_map_page(struct drm_i915_gem_object *obj,
276				      enum i915_map_type type)
277{
278	unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i;
279	struct page *stack[32], **pages = stack, *page;
280	struct sgt_iter iter;
281	pgprot_t pgprot;
282	void *vaddr;
283
284	switch (type) {
285	default:
286		MISSING_CASE(type);
287		fallthrough;	/* to use PAGE_KERNEL anyway */
288	case I915_MAP_WB:
289		/*
290		 * On 32b, highmem using a finite set of indirect PTE (i.e.
291		 * vmap) to provide virtual mappings of the high pages.
292		 * As these are finite, map_new_virtual() must wait for some
293		 * other kmap() to finish when it runs out. If we map a large
294		 * number of objects, there is no method for it to tell us
295		 * to release the mappings, and we deadlock.
296		 *
297		 * However, if we make an explicit vmap of the page, that
298		 * uses a larger vmalloc arena, and also has the ability
299		 * to tell us to release unwanted mappings. Most importantly,
300		 * it will fail and propagate an error instead of waiting
301		 * forever.
302		 *
303		 * So if the page is beyond the 32b boundary, make an explicit
304		 * vmap.
305		 */
306		if (n_pages == 1 && !PageHighMem(sg_page(obj->mm.pages->sgl)))
307			return page_address(sg_page(obj->mm.pages->sgl));
308		pgprot = PAGE_KERNEL;
309		break;
310	case I915_MAP_WC:
311		pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
312		break;
313	}
314
315	if (n_pages > ARRAY_SIZE(stack)) {
316		/* Too big for stack -- allocate temporary array instead */
317		pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL);
318		if (!pages)
319			return ERR_PTR(-ENOMEM);
320	}
321
322	i = 0;
323	for_each_sgt_page(page, iter, obj->mm.pages)
324		pages[i++] = page;
325	vaddr = vmap(pages, n_pages, 0, pgprot);
326	if (pages != stack)
327		kvfree(pages);
328
329	return vaddr ?: ERR_PTR(-ENOMEM);
330}
331
332static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj,
333				     enum i915_map_type type)
334{
335	resource_size_t iomap = obj->mm.region->iomap.base -
336		obj->mm.region->region.start;
337	unsigned long n_pfn = obj->base.size >> PAGE_SHIFT;
338	unsigned long stack[32], *pfns = stack, i;
339	struct sgt_iter iter;
340	dma_addr_t addr;
341	void *vaddr;
342
343	GEM_BUG_ON(type != I915_MAP_WC);
344
345	if (n_pfn > ARRAY_SIZE(stack)) {
346		/* Too big for stack -- allocate temporary array instead */
347		pfns = kvmalloc_array(n_pfn, sizeof(*pfns), GFP_KERNEL);
348		if (!pfns)
349			return ERR_PTR(-ENOMEM);
350	}
351
352	i = 0;
353	for_each_sgt_daddr(addr, iter, obj->mm.pages)
354		pfns[i++] = (iomap + addr) >> PAGE_SHIFT;
355	vaddr = vmap_pfn(pfns, n_pfn, pgprot_writecombine(PAGE_KERNEL_IO));
356	if (pfns != stack)
357		kvfree(pfns);
358
359	return vaddr ?: ERR_PTR(-ENOMEM);
360}
361
362struct intel_panic {
363	struct page **pages;
364	int page;
365	void *vaddr;
366};
367
368static void i915_panic_kunmap(struct intel_panic *panic)
369{
370	if (panic->vaddr) {
371		drm_clflush_virt_range(panic->vaddr, PAGE_SIZE);
372		kunmap_local(panic->vaddr);
373		panic->vaddr = NULL;
374	}
375}
376
377static struct page **i915_gem_object_panic_pages(struct drm_i915_gem_object *obj)
378{
379	unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i;
380	struct page *page;
381	struct page **pages;
382	struct sgt_iter iter;
383
384	/* For a 3840x2160 32 bits Framebuffer, this should require ~64K */
385	pages = kmalloc_array(n_pages, sizeof(*pages), GFP_ATOMIC);
386	if (!pages)
387		return NULL;
388
389	i = 0;
390	for_each_sgt_page(page, iter, obj->mm.pages)
391		pages[i++] = page;
392	return pages;
393}
394
395static void i915_gem_object_panic_map_set_pixel(struct drm_scanout_buffer *sb, unsigned int x,
396						unsigned int y, u32 color)
397{
398	struct intel_framebuffer *fb = (struct intel_framebuffer *)sb->private;
399	unsigned int offset = fb->panic_tiling(sb->width, x, y);
400
401	iosys_map_wr(&sb->map[0], offset, u32, color);
402}
403
404/*
405 * The scanout buffer pages are not mapped, so for each pixel,
406 * use kmap_local_page_try_from_panic() to map the page, and write the pixel.
407 * Try to keep the map from the previous pixel, to avoid too much map/unmap.
408 */
409static void i915_gem_object_panic_page_set_pixel(struct drm_scanout_buffer *sb, unsigned int x,
410						 unsigned int y, u32 color)
411{
412	unsigned int new_page;
413	unsigned int offset;
414	struct intel_framebuffer *fb = (struct intel_framebuffer *)sb->private;
415	struct intel_panic *panic = fb->panic;
416
417	if (fb->panic_tiling)
418		offset = fb->panic_tiling(sb->width, x, y);
419	else
420		offset = y * sb->pitch[0] + x * sb->format->cpp[0];
421
422	new_page = offset >> PAGE_SHIFT;
423	offset = offset % PAGE_SIZE;
424	if (new_page != panic->page) {
425		i915_panic_kunmap(panic);
426		panic->page = new_page;
427		panic->vaddr =
428			kmap_local_page_try_from_panic(panic->pages[panic->page]);
429	}
430	if (panic->vaddr) {
431		u32 *pix = panic->vaddr + offset;
432		*pix = color;
433	}
434}
435
436struct intel_panic *i915_gem_object_alloc_panic(void)
437{
438	struct intel_panic *panic;
439
440	panic = kzalloc(sizeof(*panic), GFP_KERNEL);
441
442	return panic;
443}
444
445/*
446 * Setup the gem framebuffer for drm_panic access.
447 * Use current vaddr if it exists, or setup a list of pages.
448 * pfn is not supported yet.
449 */
450int i915_gem_object_panic_setup(struct intel_panic *panic, struct drm_scanout_buffer *sb,
451				struct drm_gem_object *_obj, bool panic_tiling)
452{
453	enum i915_map_type has_type;
454	struct drm_i915_gem_object *obj = to_intel_bo(_obj);
455	void *ptr;
456
457	ptr = page_unpack_bits(obj->mm.mapping, &has_type);
458	if (ptr) {
459		if (i915_gem_object_has_iomem(obj))
460			iosys_map_set_vaddr_iomem(&sb->map[0], (void __iomem *)ptr);
461		else
462			iosys_map_set_vaddr(&sb->map[0], ptr);
463
464		if (panic_tiling)
465			sb->set_pixel = i915_gem_object_panic_map_set_pixel;
466		return 0;
467	}
468	if (i915_gem_object_has_struct_page(obj)) {
469		panic->pages = i915_gem_object_panic_pages(obj);
470		if (!panic->pages)
471			return -ENOMEM;
472		panic->page = -1;
473		sb->set_pixel = i915_gem_object_panic_page_set_pixel;
474		return 0;
475	}
476	return -EOPNOTSUPP;
477}
478
479void i915_gem_object_panic_finish(struct intel_panic *panic)
480{
481	i915_panic_kunmap(panic);
482	panic->page = -1;
483	kfree(panic->pages);
484	panic->pages = NULL;
485}
486
487/* get, pin, and map the pages of the object into kernel space */
488void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
489			      enum i915_map_type type)
490{
491	enum i915_map_type has_type;
492	bool pinned;
493	void *ptr;
494	int err;
495
496	if (!i915_gem_object_has_struct_page(obj) &&
497	    !i915_gem_object_has_iomem(obj))
498		return ERR_PTR(-ENXIO);
499
500	if (WARN_ON_ONCE(obj->flags & I915_BO_ALLOC_GPU_ONLY))
501		return ERR_PTR(-EINVAL);
502
503	assert_object_held(obj);
504
505	pinned = !(type & I915_MAP_OVERRIDE);
506	type &= ~I915_MAP_OVERRIDE;
507
508	if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
509		if (unlikely(!i915_gem_object_has_pages(obj))) {
510			GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
511
512			err = ____i915_gem_object_get_pages(obj);
513			if (err)
514				return ERR_PTR(err);
515
516			smp_mb__before_atomic();
517		}
518		atomic_inc(&obj->mm.pages_pin_count);
519		pinned = false;
520	}
521	GEM_BUG_ON(!i915_gem_object_has_pages(obj));
522
523	/*
524	 * For discrete our CPU mappings needs to be consistent in order to
525	 * function correctly on !x86. When mapping things through TTM, we use
526	 * the same rules to determine the caching type.
527	 *
528	 * The caching rules, starting from DG1:
529	 *
530	 *	- If the object can be placed in device local-memory, then the
531	 *	  pages should be allocated and mapped as write-combined only.
532	 *
533	 *	- Everything else is always allocated and mapped as write-back,
534	 *	  with the guarantee that everything is also coherent with the
535	 *	  GPU.
536	 *
537	 * Internal users of lmem are already expected to get this right, so no
538	 * fudging needed there.
539	 */
540	if (i915_gem_object_placement_possible(obj, INTEL_MEMORY_LOCAL)) {
541		if (type != I915_MAP_WC && !obj->mm.n_placements) {
542			ptr = ERR_PTR(-ENODEV);
543			goto err_unpin;
544		}
545
546		type = I915_MAP_WC;
547	} else if (IS_DGFX(to_i915(obj->base.dev))) {
548		type = I915_MAP_WB;
549	}
550
551	ptr = page_unpack_bits(obj->mm.mapping, &has_type);
552	if (ptr && has_type != type) {
553		if (pinned) {
554			ptr = ERR_PTR(-EBUSY);
555			goto err_unpin;
556		}
557
558		unmap_object(obj, ptr);
559
560		ptr = obj->mm.mapping = NULL;
561	}
562
563	if (!ptr) {
564		err = i915_gem_object_wait_moving_fence(obj, true);
565		if (err) {
566			ptr = ERR_PTR(err);
567			goto err_unpin;
568		}
569
570		if (GEM_WARN_ON(type == I915_MAP_WC && !pat_enabled()))
571			ptr = ERR_PTR(-ENODEV);
572		else if (i915_gem_object_has_struct_page(obj))
573			ptr = i915_gem_object_map_page(obj, type);
574		else
575			ptr = i915_gem_object_map_pfn(obj, type);
576		if (IS_ERR(ptr))
577			goto err_unpin;
578
579		obj->mm.mapping = page_pack_bits(ptr, type);
580	}
581
582	return ptr;
583
584err_unpin:
585	atomic_dec(&obj->mm.pages_pin_count);
586	return ptr;
587}
588
589void *i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj,
590				       enum i915_map_type type)
591{
592	void *ret;
593
594	i915_gem_object_lock(obj, NULL);
595	ret = i915_gem_object_pin_map(obj, type);
596	i915_gem_object_unlock(obj);
597
598	return ret;
599}
600
601void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
602				 unsigned long offset,
603				 unsigned long size)
604{
605	enum i915_map_type has_type;
606	void *ptr;
607
608	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
609	GEM_BUG_ON(range_overflows_t(typeof(obj->base.size),
610				     offset, size, obj->base.size));
611
612	wmb(); /* let all previous writes be visible to coherent partners */
613	obj->mm.dirty = true;
614
615	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)
616		return;
617
618	ptr = page_unpack_bits(obj->mm.mapping, &has_type);
619	if (has_type == I915_MAP_WC)
620		return;
621
622	drm_clflush_virt_range(ptr + offset, size);
623	if (size == obj->base.size) {
624		obj->write_domain &= ~I915_GEM_DOMAIN_CPU;
625		obj->cache_dirty = false;
626	}
627}
628
629void __i915_gem_object_release_map(struct drm_i915_gem_object *obj)
630{
631	GEM_BUG_ON(!obj->mm.mapping);
632
633	/*
634	 * We allow removing the mapping from underneath pinned pages!
635	 *
636	 * Furthermore, since this is an unsafe operation reserved only
637	 * for construction time manipulation, we ignore locking prudence.
638	 */
639	unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping)));
640
641	i915_gem_object_unpin_map(obj);
642}
643
644struct scatterlist *
645__i915_gem_object_page_iter_get_sg(struct drm_i915_gem_object *obj,
646				   struct i915_gem_object_page_iter *iter,
647				   pgoff_t n,
648				   unsigned int *offset)
649
650{
651	const bool dma = iter == &obj->mm.get_dma_page ||
652			 iter == &obj->ttm.get_io_page;
653	unsigned int idx, count;
654	struct scatterlist *sg;
655
656	might_sleep();
657	GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
658	if (!i915_gem_object_has_pinned_pages(obj))
659		assert_object_held(obj);
660
661	/* As we iterate forward through the sg, we record each entry in a
662	 * radixtree for quick repeated (backwards) lookups. If we have seen
663	 * this index previously, we will have an entry for it.
664	 *
665	 * Initial lookup is O(N), but this is amortized to O(1) for
666	 * sequential page access (where each new request is consecutive
667	 * to the previous one). Repeated lookups are O(lg(obj->base.size)),
668	 * i.e. O(1) with a large constant!
669	 */
670	if (n < READ_ONCE(iter->sg_idx))
671		goto lookup;
672
673	mutex_lock(&iter->lock);
674
675	/* We prefer to reuse the last sg so that repeated lookup of this
676	 * (or the subsequent) sg are fast - comparing against the last
677	 * sg is faster than going through the radixtree.
678	 */
679
680	sg = iter->sg_pos;
681	idx = iter->sg_idx;
682	count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
683
684	while (idx + count <= n) {
685		void *entry;
686		unsigned long i;
687		int ret;
688
689		/* If we cannot allocate and insert this entry, or the
690		 * individual pages from this range, cancel updating the
691		 * sg_idx so that on this lookup we are forced to linearly
692		 * scan onwards, but on future lookups we will try the
693		 * insertion again (in which case we need to be careful of
694		 * the error return reporting that we have already inserted
695		 * this index).
696		 */
697		ret = radix_tree_insert(&iter->radix, idx, sg);
698		if (ret && ret != -EEXIST)
699			goto scan;
700
701		entry = xa_mk_value(idx);
702		for (i = 1; i < count; i++) {
703			ret = radix_tree_insert(&iter->radix, idx + i, entry);
704			if (ret && ret != -EEXIST)
705				goto scan;
706		}
707
708		idx += count;
709		sg = ____sg_next(sg);
710		count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
711	}
712
713scan:
714	iter->sg_pos = sg;
715	iter->sg_idx = idx;
716
717	mutex_unlock(&iter->lock);
718
719	if (unlikely(n < idx)) /* insertion completed by another thread */
720		goto lookup;
721
722	/* In case we failed to insert the entry into the radixtree, we need
723	 * to look beyond the current sg.
724	 */
725	while (idx + count <= n) {
726		idx += count;
727		sg = ____sg_next(sg);
728		count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
729	}
730
731	*offset = n - idx;
732	return sg;
733
734lookup:
735	rcu_read_lock();
736
737	sg = radix_tree_lookup(&iter->radix, n);
738	GEM_BUG_ON(!sg);
739
740	/* If this index is in the middle of multi-page sg entry,
741	 * the radix tree will contain a value entry that points
742	 * to the start of that range. We will return the pointer to
743	 * the base page and the offset of this page within the
744	 * sg entry's range.
745	 */
746	*offset = 0;
747	if (unlikely(xa_is_value(sg))) {
748		unsigned long base = xa_to_value(sg);
749
750		sg = radix_tree_lookup(&iter->radix, base);
751		GEM_BUG_ON(!sg);
752
753		*offset = n - base;
754	}
755
756	rcu_read_unlock();
757
758	return sg;
759}
760
761struct page *
762__i915_gem_object_get_page(struct drm_i915_gem_object *obj, pgoff_t n)
763{
764	struct scatterlist *sg;
765	unsigned int offset;
766
767	GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
768
769	sg = i915_gem_object_get_sg(obj, n, &offset);
770	return sg_page(sg) + offset;
771}
772
773/* Like i915_gem_object_get_page(), but mark the returned page dirty */
774struct page *
775__i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, pgoff_t n)
776{
777	struct page *page;
778
779	page = i915_gem_object_get_page(obj, n);
780	if (!obj->mm.dirty)
781		set_page_dirty(page);
782
783	return page;
784}
785
786dma_addr_t
787__i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj,
788				      pgoff_t n, unsigned int *len)
789{
790	struct scatterlist *sg;
791	unsigned int offset;
792
793	sg = i915_gem_object_get_sg_dma(obj, n, &offset);
794
795	if (len)
796		*len = sg_dma_len(sg) - (offset << PAGE_SHIFT);
797
798	return sg_dma_address(sg) + (offset << PAGE_SHIFT);
799}
800
801dma_addr_t
802__i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, pgoff_t n)
803{
804	return i915_gem_object_get_dma_address_len(obj, n, NULL);
805}