lm.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright(c) 2023 Advanced Micro Devices, Inc. */
  3
  4#include <linux/anon_inodes.h>
  5#include <linux/file.h>
  6#include <linux/fs.h>
  7#include <linux/highmem.h>
  8#include <linux/vfio.h>
  9#include <linux/vfio_pci_core.h>
 10
 11#include "vfio_dev.h"
 12#include "cmds.h"
 13
 14static struct pds_vfio_lm_file *
 15pds_vfio_get_lm_file(const struct file_operations *fops, int flags, u64 size)
 16{
 17	struct pds_vfio_lm_file *lm_file = NULL;
 18	unsigned long long npages;
 19	struct page **pages;
 20	void *page_mem;
 21	const void *p;
 22
 23	if (!size)
 24		return NULL;
 25
 26	/* Alloc file structure */
 27	lm_file = kzalloc(sizeof(*lm_file), GFP_KERNEL);
 28	if (!lm_file)
 29		return NULL;
 30
 31	/* Create file */
 32	lm_file->filep =
 33		anon_inode_getfile("pds_vfio_lm", fops, lm_file, flags);
 34	if (IS_ERR(lm_file->filep))
 35		goto out_free_file;
 36
 37	stream_open(lm_file->filep->f_inode, lm_file->filep);
 38	mutex_init(&lm_file->lock);
 39
 40	/* prevent file from being released before we are done with it */
 41	get_file(lm_file->filep);
 42
 43	/* Allocate memory for file pages */
 44	npages = DIV_ROUND_UP_ULL(size, PAGE_SIZE);
 45	pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
 46	if (!pages)
 47		goto out_put_file;
 48
 49	page_mem = kvzalloc(ALIGN(size, PAGE_SIZE), GFP_KERNEL);
 50	if (!page_mem)
 51		goto out_free_pages_array;
 52
 53	p = page_mem - offset_in_page(page_mem);
 54	for (unsigned long long i = 0; i < npages; i++) {
 55		if (is_vmalloc_addr(p))
 56			pages[i] = vmalloc_to_page(p);
 57		else
 58			pages[i] = kmap_to_page((void *)p);
 59		if (!pages[i])
 60			goto out_free_page_mem;
 61
 62		p += PAGE_SIZE;
 63	}
 64
 65	/* Create scatterlist of file pages to use for DMA mapping later */
 66	if (sg_alloc_table_from_pages(&lm_file->sg_table, pages, npages, 0,
 67				      size, GFP_KERNEL))
 68		goto out_free_page_mem;
 69
 70	lm_file->size = size;
 71	lm_file->pages = pages;
 72	lm_file->npages = npages;
 73	lm_file->page_mem = page_mem;
 74	lm_file->alloc_size = npages * PAGE_SIZE;
 75
 76	return lm_file;
 77
 78out_free_page_mem:
 79	kvfree(page_mem);
 80out_free_pages_array:
 81	kfree(pages);
 82out_put_file:
 83	fput(lm_file->filep);
 84	mutex_destroy(&lm_file->lock);
 85out_free_file:
 86	kfree(lm_file);
 87
 88	return NULL;
 89}
 90
 91static void pds_vfio_put_lm_file(struct pds_vfio_lm_file *lm_file)
 92{
 93	mutex_lock(&lm_file->lock);
 94
 95	lm_file->disabled = true;
 96	lm_file->size = 0;
 97	lm_file->alloc_size = 0;
 98	lm_file->filep->f_pos = 0;
 99
100	/* Free scatter list of file pages */
101	sg_free_table(&lm_file->sg_table);
102
103	kvfree(lm_file->page_mem);
104	lm_file->page_mem = NULL;
105	kfree(lm_file->pages);
106	lm_file->pages = NULL;
107
108	mutex_unlock(&lm_file->lock);
109
110	/* allow file to be released since we are done with it */
111	fput(lm_file->filep);
112}
113
114void pds_vfio_put_save_file(struct pds_vfio_pci_device *pds_vfio)
115{
116	if (!pds_vfio->save_file)
117		return;
118
119	pds_vfio_put_lm_file(pds_vfio->save_file);
120	pds_vfio->save_file = NULL;
121}
122
123void pds_vfio_put_restore_file(struct pds_vfio_pci_device *pds_vfio)
124{
125	if (!pds_vfio->restore_file)
126		return;
127
128	pds_vfio_put_lm_file(pds_vfio->restore_file);
129	pds_vfio->restore_file = NULL;
130}
131
132static struct page *pds_vfio_get_file_page(struct pds_vfio_lm_file *lm_file,
133					   unsigned long offset)
134{
135	unsigned long cur_offset = 0;
136	struct scatterlist *sg;
137	unsigned int i;
138
139	/* All accesses are sequential */
140	if (offset < lm_file->last_offset || !lm_file->last_offset_sg) {
141		lm_file->last_offset = 0;
142		lm_file->last_offset_sg = lm_file->sg_table.sgl;
143		lm_file->sg_last_entry = 0;
144	}
145
146	cur_offset = lm_file->last_offset;
147
148	for_each_sg(lm_file->last_offset_sg, sg,
149		    lm_file->sg_table.orig_nents - lm_file->sg_last_entry, i) {
150		if (offset < sg->length + cur_offset) {
151			lm_file->last_offset_sg = sg;
152			lm_file->sg_last_entry += i;
153			lm_file->last_offset = cur_offset;
154			return sg_page(sg) + (offset - cur_offset) / PAGE_SIZE;
155		}
156		cur_offset += sg->length;
157	}
158
159	return NULL;
160}
161
162static int pds_vfio_release_file(struct inode *inode, struct file *filp)
163{
164	struct pds_vfio_lm_file *lm_file = filp->private_data;
165
166	mutex_lock(&lm_file->lock);
167	lm_file->filep->f_pos = 0;
168	lm_file->size = 0;
169	mutex_unlock(&lm_file->lock);
170	mutex_destroy(&lm_file->lock);
171	kfree(lm_file);
172
173	return 0;
174}
175
176static ssize_t pds_vfio_save_read(struct file *filp, char __user *buf,
177				  size_t len, loff_t *pos)
178{
179	struct pds_vfio_lm_file *lm_file = filp->private_data;
180	ssize_t done = 0;
181
182	if (pos)
183		return -ESPIPE;
184	pos = &filp->f_pos;
185
186	mutex_lock(&lm_file->lock);
187
188	if (lm_file->disabled) {
189		done = -ENODEV;
190		goto out_unlock;
191	}
192
193	if (*pos > lm_file->size) {
194		done = -EINVAL;
195		goto out_unlock;
196	}
197
198	len = min_t(size_t, lm_file->size - *pos, len);
199	while (len) {
200		size_t page_offset;
201		struct page *page;
202		size_t page_len;
203		u8 *from_buff;
204		int err;
205
206		page_offset = (*pos) % PAGE_SIZE;
207		page = pds_vfio_get_file_page(lm_file, *pos - page_offset);
208		if (!page) {
209			if (done == 0)
210				done = -EINVAL;
211			goto out_unlock;
212		}
213
214		page_len = min_t(size_t, len, PAGE_SIZE - page_offset);
215		from_buff = kmap_local_page(page);
216		err = copy_to_user(buf, from_buff + page_offset, page_len);
217		kunmap_local(from_buff);
218		if (err) {
219			done = -EFAULT;
220			goto out_unlock;
221		}
222		*pos += page_len;
223		len -= page_len;
224		done += page_len;
225		buf += page_len;
226	}
227
228out_unlock:
229	mutex_unlock(&lm_file->lock);
230	return done;
231}
232
233static const struct file_operations pds_vfio_save_fops = {
234	.owner = THIS_MODULE,
235	.read = pds_vfio_save_read,
236	.release = pds_vfio_release_file,
237};
238
239static int pds_vfio_get_save_file(struct pds_vfio_pci_device *pds_vfio)
240{
241	struct device *dev = &pds_vfio->vfio_coredev.pdev->dev;
242	struct pds_vfio_lm_file *lm_file;
243	u64 size;
244	int err;
245
246	/* Get live migration state size in this state */
247	err = pds_vfio_get_lm_state_size_cmd(pds_vfio, &size);
248	if (err) {
249		dev_err(dev, "failed to get save status: %pe\n", ERR_PTR(err));
250		return err;
251	}
252
253	dev_dbg(dev, "save status, size = %lld\n", size);
254
255	if (!size) {
256		dev_err(dev, "invalid state size\n");
257		return -EIO;
258	}
259
260	lm_file = pds_vfio_get_lm_file(&pds_vfio_save_fops, O_RDONLY, size);
261	if (!lm_file) {
262		dev_err(dev, "failed to create save file\n");
263		return -ENOENT;
264	}
265
266	dev_dbg(dev, "size = %lld, alloc_size = %lld, npages = %lld\n",
267		lm_file->size, lm_file->alloc_size, lm_file->npages);
268
269	pds_vfio->save_file = lm_file;
270
271	return 0;
272}
273
274static ssize_t pds_vfio_restore_write(struct file *filp, const char __user *buf,
275				      size_t len, loff_t *pos)
276{
277	struct pds_vfio_lm_file *lm_file = filp->private_data;
278	loff_t requested_length;
279	ssize_t done = 0;
280
281	if (pos)
282		return -ESPIPE;
283
284	pos = &filp->f_pos;
285
286	if (*pos < 0 ||
287	    check_add_overflow((loff_t)len, *pos, &requested_length))
288		return -EINVAL;
289
290	mutex_lock(&lm_file->lock);
291
292	if (lm_file->disabled) {
293		done = -ENODEV;
294		goto out_unlock;
295	}
296
297	while (len) {
298		size_t page_offset;
299		struct page *page;
300		size_t page_len;
301		u8 *to_buff;
302		int err;
303
304		page_offset = (*pos) % PAGE_SIZE;
305		page = pds_vfio_get_file_page(lm_file, *pos - page_offset);
306		if (!page) {
307			if (done == 0)
308				done = -EINVAL;
309			goto out_unlock;
310		}
311
312		page_len = min_t(size_t, len, PAGE_SIZE - page_offset);
313		to_buff = kmap_local_page(page);
314		err = copy_from_user(to_buff + page_offset, buf, page_len);
315		kunmap_local(to_buff);
316		if (err) {
317			done = -EFAULT;
318			goto out_unlock;
319		}
320		*pos += page_len;
321		len -= page_len;
322		done += page_len;
323		buf += page_len;
324		lm_file->size += page_len;
325	}
326out_unlock:
327	mutex_unlock(&lm_file->lock);
328	return done;
329}
330
331static const struct file_operations pds_vfio_restore_fops = {
332	.owner = THIS_MODULE,
333	.write = pds_vfio_restore_write,
334	.release = pds_vfio_release_file,
335};
336
337static int pds_vfio_get_restore_file(struct pds_vfio_pci_device *pds_vfio)
338{
339	struct device *dev = &pds_vfio->vfio_coredev.pdev->dev;
340	struct pds_vfio_lm_file *lm_file;
341	u64 size;
342
343	size = sizeof(union pds_lm_dev_state);
344	dev_dbg(dev, "restore status, size = %lld\n", size);
345
346	if (!size) {
347		dev_err(dev, "invalid state size");
348		return -EIO;
349	}
350
351	lm_file = pds_vfio_get_lm_file(&pds_vfio_restore_fops, O_WRONLY, size);
352	if (!lm_file) {
353		dev_err(dev, "failed to create restore file");
354		return -ENOENT;
355	}
356	pds_vfio->restore_file = lm_file;
357
358	return 0;
359}
360
361struct file *
362pds_vfio_step_device_state_locked(struct pds_vfio_pci_device *pds_vfio,
363				  enum vfio_device_mig_state next)
364{
365	enum vfio_device_mig_state cur = pds_vfio->state;
366	int err;
367
368	if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_STOP_COPY) {
369		err = pds_vfio_get_save_file(pds_vfio);
370		if (err)
371			return ERR_PTR(err);
372
373		err = pds_vfio_get_lm_state_cmd(pds_vfio);
374		if (err) {
375			pds_vfio_put_save_file(pds_vfio);
376			return ERR_PTR(err);
377		}
378
379		return pds_vfio->save_file->filep;
380	}
381
382	if (cur == VFIO_DEVICE_STATE_STOP_COPY && next == VFIO_DEVICE_STATE_STOP) {
383		pds_vfio_put_save_file(pds_vfio);
384		pds_vfio_dirty_disable(pds_vfio, true);
385		return NULL;
386	}
387
388	if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_RESUMING) {
389		err = pds_vfio_get_restore_file(pds_vfio);
390		if (err)
391			return ERR_PTR(err);
392
393		return pds_vfio->restore_file->filep;
394	}
395
396	if (cur == VFIO_DEVICE_STATE_RESUMING && next == VFIO_DEVICE_STATE_STOP) {
397		err = pds_vfio_set_lm_state_cmd(pds_vfio);
398		if (err)
399			return ERR_PTR(err);
400
401		pds_vfio_put_restore_file(pds_vfio);
402		return NULL;
403	}
404
405	if (cur == VFIO_DEVICE_STATE_RUNNING && next == VFIO_DEVICE_STATE_RUNNING_P2P) {
406		pds_vfio_send_host_vf_lm_status_cmd(pds_vfio,
407						    PDS_LM_STA_IN_PROGRESS);
408		err = pds_vfio_suspend_device_cmd(pds_vfio,
409						  PDS_LM_SUSPEND_RESUME_TYPE_P2P);
410		if (err)
411			return ERR_PTR(err);
412
413		return NULL;
414	}
415
416	if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && next == VFIO_DEVICE_STATE_RUNNING) {
417		err = pds_vfio_resume_device_cmd(pds_vfio,
418						 PDS_LM_SUSPEND_RESUME_TYPE_FULL);
419		if (err)
420			return ERR_PTR(err);
421
422		pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, PDS_LM_STA_NONE);
423		return NULL;
424	}
425
426	if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_RUNNING_P2P) {
427		err = pds_vfio_resume_device_cmd(pds_vfio,
428						 PDS_LM_SUSPEND_RESUME_TYPE_P2P);
429		if (err)
430			return ERR_PTR(err);
431
432		return NULL;
433	}
434
435	if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && next == VFIO_DEVICE_STATE_STOP) {
436		err = pds_vfio_suspend_device_cmd(pds_vfio,
437						  PDS_LM_SUSPEND_RESUME_TYPE_FULL);
438		if (err)
439			return ERR_PTR(err);
440		return NULL;
441	}
442
443	return ERR_PTR(-EINVAL);
444}