vmwgfx_page_dirty.c

  1// SPDX-License-Identifier: GPL-2.0 OR MIT
  2/**************************************************************************
  3 *
  4 * Copyright (c) 2019-2025 Broadcom. All Rights Reserved. The term
  5 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
  6 *
  7 **************************************************************************/
  8#include "vmwgfx_bo.h"
  9#include "vmwgfx_drv.h"
 10
 11/*
 12 * Different methods for tracking dirty:
 13 * VMW_BO_DIRTY_PAGETABLE - Scan the pagetable for hardware dirty bits
 14 * VMW_BO_DIRTY_MKWRITE - Write-protect page table entries and record write-
 15 * accesses in the VM mkwrite() callback
 16 */
 17enum vmw_bo_dirty_method {
 18	VMW_BO_DIRTY_PAGETABLE,
 19	VMW_BO_DIRTY_MKWRITE,
 20};
 21
 22/*
 23 * No dirtied pages at scan trigger a transition to the _MKWRITE method,
 24 * similarly a certain percentage of dirty pages trigger a transition to
 25 * the _PAGETABLE method. How many triggers should we wait for before
 26 * changing method?
 27 */
 28#define VMW_DIRTY_NUM_CHANGE_TRIGGERS 2
 29
 30/* Percentage to trigger a transition to the _PAGETABLE method */
 31#define VMW_DIRTY_PERCENTAGE 10
 32
 33/**
 34 * struct vmw_bo_dirty - Dirty information for buffer objects
 35 * @ref_count: Reference count for this structure. Must be first member!
 36 * @start: First currently dirty bit
 37 * @end: Last currently dirty bit + 1
 38 * @method: The currently used dirty method
 39 * @change_count: Number of consecutive method change triggers
 40 * @bitmap_size: The size of the bitmap in bits. Typically equal to the
 41 * nuber of pages in the bo.
 42 * @bitmap: A bitmap where each bit represents a page. A set bit means a
 43 * dirty page.
 44 */
 45struct vmw_bo_dirty {
 46	struct   kref ref_count;
 47	unsigned long start;
 48	unsigned long end;
 49	enum vmw_bo_dirty_method method;
 50	unsigned int change_count;
 51	unsigned long bitmap_size;
 52	unsigned long bitmap[];
 53};
 54
 55bool vmw_bo_is_dirty(struct vmw_bo *vbo)
 56{
 57	return vbo->dirty && (vbo->dirty->start < vbo->dirty->end);
 58}
 59
 60/**
 61 * vmw_bo_dirty_scan_pagetable - Perform a pagetable scan for dirty bits
 62 * @vbo: The buffer object to scan
 63 *
 64 * Scans the pagetable for dirty bits. Clear those bits and modify the
 65 * dirty structure with the results. This function may change the
 66 * dirty-tracking method.
 67 */
 68static void vmw_bo_dirty_scan_pagetable(struct vmw_bo *vbo)
 69{
 70	struct vmw_bo_dirty *dirty = vbo->dirty;
 71	pgoff_t offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
 72	struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
 73	pgoff_t num_marked;
 74
 75	num_marked = clean_record_shared_mapping_range
 76		(mapping,
 77		 offset, dirty->bitmap_size,
 78		 offset, &dirty->bitmap[0],
 79		 &dirty->start, &dirty->end);
 80	if (num_marked == 0)
 81		dirty->change_count++;
 82	else
 83		dirty->change_count = 0;
 84
 85	if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {
 86		dirty->change_count = 0;
 87		dirty->method = VMW_BO_DIRTY_MKWRITE;
 88		wp_shared_mapping_range(mapping,
 89					offset, dirty->bitmap_size);
 90		clean_record_shared_mapping_range(mapping,
 91						  offset, dirty->bitmap_size,
 92						  offset, &dirty->bitmap[0],
 93						  &dirty->start, &dirty->end);
 94	}
 95}
 96
 97/**
 98 * vmw_bo_dirty_scan_mkwrite - Reset the mkwrite dirty-tracking method
 99 * @vbo: The buffer object to scan
100 *
101 * Write-protect pages written to so that consecutive write accesses will
102 * trigger a call to mkwrite.
103 *
104 * This function may change the dirty-tracking method.
105 */
106static void vmw_bo_dirty_scan_mkwrite(struct vmw_bo *vbo)
107{
108	struct vmw_bo_dirty *dirty = vbo->dirty;
109	unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
110	struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
111	pgoff_t num_marked;
112
113	if (dirty->end <= dirty->start)
114		return;
115
116	num_marked = wp_shared_mapping_range(vbo->tbo.bdev->dev_mapping,
117					     dirty->start + offset,
118					     dirty->end - dirty->start);
119
120	if (100UL * num_marked / dirty->bitmap_size >
121	    VMW_DIRTY_PERCENTAGE)
122		dirty->change_count++;
123	else
124		dirty->change_count = 0;
125
126	if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {
127		pgoff_t start = 0;
128		pgoff_t end = dirty->bitmap_size;
129
130		dirty->method = VMW_BO_DIRTY_PAGETABLE;
131		clean_record_shared_mapping_range(mapping, offset, end, offset,
132						  &dirty->bitmap[0],
133						  &start, &end);
134		bitmap_clear(&dirty->bitmap[0], 0, dirty->bitmap_size);
135		if (dirty->start < dirty->end)
136			bitmap_set(&dirty->bitmap[0], dirty->start,
137				   dirty->end - dirty->start);
138		dirty->change_count = 0;
139	}
140}
141
142/**
143 * vmw_bo_dirty_scan - Scan for dirty pages and add them to the dirty
144 * tracking structure
145 * @vbo: The buffer object to scan
146 *
147 * This function may change the dirty tracking method.
148 */
149void vmw_bo_dirty_scan(struct vmw_bo *vbo)
150{
151	struct vmw_bo_dirty *dirty = vbo->dirty;
152
153	if (dirty->method == VMW_BO_DIRTY_PAGETABLE)
154		vmw_bo_dirty_scan_pagetable(vbo);
155	else
156		vmw_bo_dirty_scan_mkwrite(vbo);
157}
158
159/**
160 * vmw_bo_dirty_pre_unmap - write-protect and pick up dirty pages before
161 * an unmap_mapping_range operation.
162 * @vbo: The buffer object,
163 * @start: First page of the range within the buffer object.
164 * @end: Last page of the range within the buffer object + 1.
165 *
166 * If we're using the _PAGETABLE scan method, we may leak dirty pages
167 * when calling unmap_mapping_range(). This function makes sure we pick
168 * up all dirty pages.
169 */
170static void vmw_bo_dirty_pre_unmap(struct vmw_bo *vbo,
171				   pgoff_t start, pgoff_t end)
172{
173	struct vmw_bo_dirty *dirty = vbo->dirty;
174	unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
175	struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
176
177	if (dirty->method != VMW_BO_DIRTY_PAGETABLE || start >= end)
178		return;
179
180	wp_shared_mapping_range(mapping, start + offset, end - start);
181	clean_record_shared_mapping_range(mapping, start + offset,
182					  end - start, offset,
183					  &dirty->bitmap[0], &dirty->start,
184					  &dirty->end);
185}
186
187/**
188 * vmw_bo_dirty_unmap - Clear all ptes pointing to a range within a bo
189 * @vbo: The buffer object,
190 * @start: First page of the range within the buffer object.
191 * @end: Last page of the range within the buffer object + 1.
192 *
193 * This is similar to ttm_bo_unmap_virtual() except it takes a subrange.
194 */
195void vmw_bo_dirty_unmap(struct vmw_bo *vbo,
196			pgoff_t start, pgoff_t end)
197{
198	unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
199	struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
200
201	vmw_bo_dirty_pre_unmap(vbo, start, end);
202	unmap_shared_mapping_range(mapping, (offset + start) << PAGE_SHIFT,
203				   (loff_t) (end - start) << PAGE_SHIFT);
204}
205
206/**
207 * vmw_bo_dirty_add - Add a dirty-tracking user to a buffer object
208 * @vbo: The buffer object
209 *
210 * This function registers a dirty-tracking user to a buffer object.
211 * A user can be for example a resource or a vma in a special user-space
212 * mapping.
213 *
214 * Return: Zero on success, -ENOMEM on memory allocation failure.
215 */
216int vmw_bo_dirty_add(struct vmw_bo *vbo)
217{
218	struct vmw_bo_dirty *dirty = vbo->dirty;
219	pgoff_t num_pages = PFN_UP(vbo->tbo.resource->size);
220	size_t size;
221	int ret;
222
223	if (dirty) {
224		kref_get(&dirty->ref_count);
225		return 0;
226	}
227
228	size = sizeof(*dirty) + BITS_TO_LONGS(num_pages) * sizeof(long);
229	dirty = kvzalloc(size, GFP_KERNEL);
230	if (!dirty) {
231		ret = -ENOMEM;
232		goto out_no_dirty;
233	}
234
235	dirty->bitmap_size = num_pages;
236	dirty->start = dirty->bitmap_size;
237	dirty->end = 0;
238	kref_init(&dirty->ref_count);
239	if (num_pages < PAGE_SIZE / sizeof(pte_t)) {
240		dirty->method = VMW_BO_DIRTY_PAGETABLE;
241	} else {
242		struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
243		pgoff_t offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
244
245		dirty->method = VMW_BO_DIRTY_MKWRITE;
246
247		/* Write-protect and then pick up already dirty bits */
248		wp_shared_mapping_range(mapping, offset, num_pages);
249		clean_record_shared_mapping_range(mapping, offset, num_pages,
250						  offset,
251						  &dirty->bitmap[0],
252						  &dirty->start, &dirty->end);
253	}
254
255	vbo->dirty = dirty;
256
257	return 0;
258
259out_no_dirty:
260	return ret;
261}
262
263static void vmw_bo_dirty_free(struct kref *kref)
264{
265	struct vmw_bo_dirty *dirty = container_of(kref, struct vmw_bo_dirty, ref_count);
266
267	kvfree(dirty);
268}
269
270/**
271 * vmw_bo_dirty_release - Release a dirty-tracking user from a buffer object
272 * @vbo: The buffer object
273 *
274 * This function releases a dirty-tracking user from a buffer object.
275 * If the reference count reaches zero, then the dirty-tracking object is
276 * freed and the pointer to it cleared.
277 *
278 * Return: Zero on success, -ENOMEM on memory allocation failure.
279 */
280void vmw_bo_dirty_release(struct vmw_bo *vbo)
281{
282	struct vmw_bo_dirty *dirty = vbo->dirty;
283
284	if (dirty && kref_put(&dirty->ref_count, vmw_bo_dirty_free))
285		vbo->dirty = NULL;
286}
287
288/**
289 * vmw_bo_dirty_transfer_to_res - Pick up a resource's dirty region from
290 * its backing mob.
291 * @res: The resource
292 *
293 * This function will pick up all dirty ranges affecting the resource from
294 * it's backup mob, and call vmw_resource_dirty_update() once for each
295 * range. The transferred ranges will be cleared from the backing mob's
296 * dirty tracking.
297 */
298void vmw_bo_dirty_transfer_to_res(struct vmw_resource *res)
299{
300	struct vmw_bo *vbo = res->guest_memory_bo;
301	struct vmw_bo_dirty *dirty = vbo->dirty;
302	pgoff_t start, cur, end;
303	unsigned long res_start = res->guest_memory_offset;
304	unsigned long res_end = res->guest_memory_offset + res->guest_memory_size;
305
306	WARN_ON_ONCE(res_start & ~PAGE_MASK);
307	res_start >>= PAGE_SHIFT;
308	res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
309
310	if (res_start >= dirty->end || res_end <= dirty->start)
311		return;
312
313	cur = max(res_start, dirty->start);
314	res_end = max(res_end, dirty->end);
315	while (cur < res_end) {
316		unsigned long num;
317
318		start = find_next_bit(&dirty->bitmap[0], res_end, cur);
319		if (start >= res_end)
320			break;
321
322		end = find_next_zero_bit(&dirty->bitmap[0], res_end, start + 1);
323		cur = end + 1;
324		num = end - start;
325		bitmap_clear(&dirty->bitmap[0], start, num);
326		vmw_resource_dirty_update(res, start, end);
327	}
328
329	if (res_start <= dirty->start && res_end > dirty->start)
330		dirty->start = res_end;
331	if (res_start < dirty->end && res_end >= dirty->end)
332		dirty->end = res_start;
333}
334
335void vmw_bo_dirty_clear(struct vmw_bo *vbo)
336{
337	struct vmw_bo_dirty *dirty = vbo->dirty;
338	pgoff_t start, cur, end;
339	unsigned long res_start = 0;
340	unsigned long res_end = vbo->tbo.base.size;
341
342	WARN_ON_ONCE(res_start & ~PAGE_MASK);
343	res_start >>= PAGE_SHIFT;
344	res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
345
346	if (res_start >= dirty->end || res_end <= dirty->start)
347		return;
348
349	cur = max(res_start, dirty->start);
350	res_end = max(res_end, dirty->end);
351	while (cur < res_end) {
352		unsigned long num;
353
354		start = find_next_bit(&dirty->bitmap[0], res_end, cur);
355		if (start >= res_end)
356			break;
357
358		end = find_next_zero_bit(&dirty->bitmap[0], res_end, start + 1);
359		cur = end + 1;
360		num = end - start;
361		bitmap_clear(&dirty->bitmap[0], start, num);
362	}
363
364	if (res_start <= dirty->start && res_end > dirty->start)
365		dirty->start = res_end;
366	if (res_start < dirty->end && res_end >= dirty->end)
367		dirty->end = res_start;
368}
369
370/**
371 * vmw_bo_dirty_clear_res - Clear a resource's dirty region from
372 * its backing mob.
373 * @res: The resource
374 *
375 * This function will clear all dirty ranges affecting the resource from
376 * it's backup mob's dirty tracking.
377 */
378void vmw_bo_dirty_clear_res(struct vmw_resource *res)
379{
380	unsigned long res_start = res->guest_memory_offset;
381	unsigned long res_end = res->guest_memory_offset + res->guest_memory_size;
382	struct vmw_bo *vbo = res->guest_memory_bo;
383	struct vmw_bo_dirty *dirty = vbo->dirty;
384
385	res_start >>= PAGE_SHIFT;
386	res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
387
388	if (res_start >= dirty->end || res_end <= dirty->start)
389		return;
390
391	res_start = max(res_start, dirty->start);
392	res_end = min(res_end, dirty->end);
393	bitmap_clear(&dirty->bitmap[0], res_start, res_end - res_start);
394
395	if (res_start <= dirty->start && res_end > dirty->start)
396		dirty->start = res_end;
397	if (res_start < dirty->end && res_end >= dirty->end)
398		dirty->end = res_start;
399}
400
401vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf)
402{
403	struct vm_area_struct *vma = vmf->vma;
404	struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
405	    vma->vm_private_data;
406	vm_fault_t ret;
407	unsigned long page_offset;
408	unsigned int save_flags;
409	struct vmw_bo *vbo = to_vmw_bo(&bo->base);
410
411	/*
412	 * mkwrite() doesn't handle the VM_FAULT_RETRY return value correctly.
413	 * So make sure the TTM helpers are aware.
414	 */
415	save_flags = vmf->flags;
416	vmf->flags &= ~FAULT_FLAG_ALLOW_RETRY;
417	ret = ttm_bo_vm_reserve(bo, vmf);
418	vmf->flags = save_flags;
419	if (ret)
420		return ret;
421
422	page_offset = vmf->pgoff - drm_vma_node_start(&bo->base.vma_node);
423	if (unlikely(page_offset >= PFN_UP(bo->resource->size))) {
424		ret = VM_FAULT_SIGBUS;
425		goto out_unlock;
426	}
427
428	if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE &&
429	    !test_bit(page_offset, &vbo->dirty->bitmap[0])) {
430		struct vmw_bo_dirty *dirty = vbo->dirty;
431
432		__set_bit(page_offset, &dirty->bitmap[0]);
433		dirty->start = min(dirty->start, page_offset);
434		dirty->end = max(dirty->end, page_offset + 1);
435	}
436
437out_unlock:
438	dma_resv_unlock(bo->base.resv);
439	return ret;
440}
441
442vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf)
443{
444	struct vm_area_struct *vma = vmf->vma;
445	struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
446	    vma->vm_private_data;
447	struct vmw_bo *vbo = to_vmw_bo(&bo->base);
448	pgoff_t num_prefault;
449	pgprot_t prot;
450	vm_fault_t ret;
451
452	ret = ttm_bo_vm_reserve(bo, vmf);
453	if (ret)
454		return ret;
455
456	num_prefault = (vma->vm_flags & VM_RAND_READ) ? 1 :
457		TTM_BO_VM_NUM_PREFAULT;
458
459	if (vbo->dirty) {
460		pgoff_t allowed_prefault;
461		unsigned long page_offset;
462
463		page_offset = vmf->pgoff -
464			drm_vma_node_start(&bo->base.vma_node);
465		if (page_offset >= PFN_UP(bo->resource->size) ||
466		    vmw_resources_clean(vbo, page_offset,
467					page_offset + PAGE_SIZE,
468					&allowed_prefault)) {
469			ret = VM_FAULT_SIGBUS;
470			goto out_unlock;
471		}
472
473		num_prefault = min(num_prefault, allowed_prefault);
474	}
475
476	/*
477	 * If we don't track dirty using the MKWRITE method, make sure
478	 * sure the page protection is write-enabled so we don't get
479	 * a lot of unnecessary write faults.
480	 */
481	if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE)
482		prot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED);
483	else
484		prot = vm_get_page_prot(vma->vm_flags);
485
486	ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault);
487	if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
488		return ret;
489
490out_unlock:
491	dma_resv_unlock(bo->base.resv);
492
493	return ret;
494}