cdev.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
  3#include <linux/init.h>
  4#include <linux/kernel.h>
  5#include <linux/module.h>
  6#include <linux/pci.h>
  7#include <linux/device.h>
  8#include <linux/sched/task.h>
  9#include <linux/io-64-nonatomic-lo-hi.h>
 10#include <linux/cdev.h>
 11#include <linux/fs.h>
 12#include <linux/poll.h>
 13#include <linux/iommu.h>
 14#include <linux/highmem.h>
 15#include <uapi/linux/idxd.h>
 16#include <linux/xarray.h>
 17#include "registers.h"
 18#include "idxd.h"
 19
 20struct idxd_cdev_context {
 21	const char *name;
 22	dev_t devt;
 23	struct ida minor_ida;
 24};
 25
 26/*
 27 * Since user file names are global in DSA devices, define their ida's as
 28 * global to avoid conflict file names.
 29 */
 30static DEFINE_IDA(file_ida);
 31
 32/*
 33 * ictx is an array based off of accelerator types. enum idxd_type
 34 * is used as index
 35 */
 36static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
 37	{ .name = "dsa" },
 38	{ .name = "iax" }
 39};
 40
 41struct idxd_user_context {
 42	struct idxd_wq *wq;
 43	struct task_struct *task;
 44	unsigned int pasid;
 45	struct mm_struct *mm;
 46	unsigned int flags;
 47	struct iommu_sva *sva;
 48	struct idxd_dev idxd_dev;
 49	u64 counters[COUNTER_MAX];
 50	int id;
 51	pid_t pid;
 52};
 53
 54static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid);
 55static void idxd_xa_pasid_remove(struct idxd_user_context *ctx);
 56
 57static inline struct idxd_user_context *dev_to_uctx(struct device *dev)
 58{
 59	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
 60
 61	return container_of(idxd_dev, struct idxd_user_context, idxd_dev);
 62}
 63
 64static ssize_t cr_faults_show(struct device *dev, struct device_attribute *attr, char *buf)
 65{
 66	struct idxd_user_context *ctx = dev_to_uctx(dev);
 67
 68	return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULTS]);
 69}
 70static DEVICE_ATTR_RO(cr_faults);
 71
 72static ssize_t cr_fault_failures_show(struct device *dev,
 73				      struct device_attribute *attr, char *buf)
 74{
 75	struct idxd_user_context *ctx = dev_to_uctx(dev);
 76
 77	return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULT_FAILS]);
 78}
 79static DEVICE_ATTR_RO(cr_fault_failures);
 80
 81static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf)
 82{
 83	struct idxd_user_context *ctx = dev_to_uctx(dev);
 84
 85	return sysfs_emit(buf, "%u\n", ctx->pid);
 86}
 87static DEVICE_ATTR_RO(pid);
 88
 89static struct attribute *cdev_file_attributes[] = {
 90	&dev_attr_cr_faults.attr,
 91	&dev_attr_cr_fault_failures.attr,
 92	&dev_attr_pid.attr,
 93	NULL
 94};
 95
 96static umode_t cdev_file_attr_visible(struct kobject *kobj, struct attribute *a, int n)
 97{
 98	struct device *dev = container_of(kobj, typeof(*dev), kobj);
 99	struct idxd_user_context *ctx = dev_to_uctx(dev);
100	struct idxd_wq *wq = ctx->wq;
101
102	if (!wq_pasid_enabled(wq))
103		return 0;
104
105	return a->mode;
106}
107
108static const struct attribute_group cdev_file_attribute_group = {
109	.attrs = cdev_file_attributes,
110	.is_visible = cdev_file_attr_visible,
111};
112
113static const struct attribute_group *cdev_file_attribute_groups[] = {
114	&cdev_file_attribute_group,
115	NULL
116};
117
118static void idxd_file_dev_release(struct device *dev)
119{
120	struct idxd_user_context *ctx = dev_to_uctx(dev);
121	struct idxd_wq *wq = ctx->wq;
122	struct idxd_device *idxd = wq->idxd;
123	int rc;
124
125	ida_free(&file_ida, ctx->id);
126
127	/* Wait for in-flight operations to complete. */
128	if (wq_shared(wq)) {
129		idxd_device_drain_pasid(idxd, ctx->pasid);
130	} else {
131		if (device_user_pasid_enabled(idxd)) {
132			/* The wq disable in the disable pasid function will drain the wq */
133			rc = idxd_wq_disable_pasid(wq);
134			if (rc < 0)
135				dev_err(dev, "wq disable pasid failed.\n");
136		} else {
137			idxd_wq_drain(wq);
138		}
139	}
140
141	if (ctx->sva) {
142		idxd_cdev_evl_drain_pasid(wq, ctx->pasid);
143		iommu_sva_unbind_device(ctx->sva);
144		idxd_xa_pasid_remove(ctx);
145	}
146	kfree(ctx);
147	mutex_lock(&wq->wq_lock);
148	idxd_wq_put(wq);
149	mutex_unlock(&wq->wq_lock);
150}
151
152static const struct device_type idxd_cdev_file_type = {
153	.name = "idxd_file",
154	.release = idxd_file_dev_release,
155	.groups = cdev_file_attribute_groups,
156};
157
158static void idxd_cdev_dev_release(struct device *dev)
159{
160	struct idxd_cdev *idxd_cdev = dev_to_cdev(dev);
161
162	kfree(idxd_cdev);
163}
164
165static const struct device_type idxd_cdev_device_type = {
166	.name = "idxd_cdev",
167	.release = idxd_cdev_dev_release,
168};
169
170static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
171{
172	struct cdev *cdev = inode->i_cdev;
173
174	return container_of(cdev, struct idxd_cdev, cdev);
175}
176
177static inline struct idxd_wq *inode_wq(struct inode *inode)
178{
179	struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);
180
181	return idxd_cdev->wq;
182}
183
184static void idxd_xa_pasid_remove(struct idxd_user_context *ctx)
185{
186	struct idxd_wq *wq = ctx->wq;
187	void *ptr;
188
189	mutex_lock(&wq->uc_lock);
190	ptr = xa_cmpxchg(&wq->upasid_xa, ctx->pasid, ctx, NULL, GFP_KERNEL);
191	if (ptr != (void *)ctx)
192		dev_warn(&wq->idxd->pdev->dev, "xarray cmpxchg failed for pasid %u\n",
193			 ctx->pasid);
194	mutex_unlock(&wq->uc_lock);
195}
196
197void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index)
198{
199	struct idxd_user_context *ctx;
200
201	if (index >= COUNTER_MAX)
202		return;
203
204	mutex_lock(&wq->uc_lock);
205	ctx = xa_load(&wq->upasid_xa, pasid);
206	if (!ctx) {
207		mutex_unlock(&wq->uc_lock);
208		return;
209	}
210	ctx->counters[index]++;
211	mutex_unlock(&wq->uc_lock);
212}
213
214static int idxd_cdev_open(struct inode *inode, struct file *filp)
215{
216	struct idxd_user_context *ctx;
217	struct idxd_device *idxd;
218	struct idxd_wq *wq;
219	struct device *dev, *fdev;
220	int rc = 0;
221	struct iommu_sva *sva = NULL;
222	unsigned int pasid;
223	struct idxd_cdev *idxd_cdev;
224
225	wq = inode_wq(inode);
226	idxd = wq->idxd;
227	dev = &idxd->pdev->dev;
228
229	dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));
230
231	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
232	if (!ctx)
233		return -ENOMEM;
234
235	mutex_lock(&wq->wq_lock);
236
237	if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {
238		rc = -EBUSY;
239		goto failed;
240	}
241
242	ctx->wq = wq;
243	filp->private_data = ctx;
244	ctx->pid = current->pid;
245
246	if (device_user_pasid_enabled(idxd)) {
247		sva = iommu_sva_bind_device(dev, current->mm);
248		if (IS_ERR(sva)) {
249			rc = PTR_ERR(sva);
250			dev_err(dev, "pasid allocation failed: %d\n", rc);
251			goto failed;
252		}
253
254		pasid = iommu_sva_get_pasid(sva);
255		if (pasid == IOMMU_PASID_INVALID) {
256			rc = -EINVAL;
257			goto failed_get_pasid;
258		}
259
260		ctx->sva = sva;
261		ctx->pasid = pasid;
262		ctx->mm = current->mm;
263
264		mutex_lock(&wq->uc_lock);
265		rc = xa_insert(&wq->upasid_xa, pasid, ctx, GFP_KERNEL);
266		mutex_unlock(&wq->uc_lock);
267		if (rc < 0)
268			dev_warn(dev, "PASID entry already exist in xarray.\n");
269
270		if (wq_dedicated(wq)) {
271			rc = idxd_wq_set_pasid(wq, pasid);
272			if (rc < 0) {
273				dev_err(dev, "wq set pasid failed: %d\n", rc);
274				goto failed_set_pasid;
275			}
276		}
277	}
278
279	idxd_cdev = wq->idxd_cdev;
280	ctx->id = ida_alloc(&file_ida, GFP_KERNEL);
281	if (ctx->id < 0) {
282		dev_warn(dev, "ida alloc failure\n");
283		goto failed_ida;
284	}
285	ctx->idxd_dev.type  = IDXD_DEV_CDEV_FILE;
286	fdev = user_ctx_dev(ctx);
287	device_initialize(fdev);
288	fdev->parent = cdev_dev(idxd_cdev);
289	fdev->bus = &dsa_bus_type;
290	fdev->type = &idxd_cdev_file_type;
291
292	rc = dev_set_name(fdev, "file%d", ctx->id);
293	if (rc < 0) {
294		dev_warn(dev, "set name failure\n");
295		goto failed_dev_name;
296	}
297
298	rc = device_add(fdev);
299	if (rc < 0) {
300		dev_warn(dev, "file device add failure\n");
301		goto failed_dev_add;
302	}
303
304	idxd_wq_get(wq);
305	mutex_unlock(&wq->wq_lock);
306	return 0;
307
308failed_dev_add:
309failed_dev_name:
310	put_device(fdev);
311failed_ida:
312failed_set_pasid:
313	if (device_user_pasid_enabled(idxd))
314		idxd_xa_pasid_remove(ctx);
315failed_get_pasid:
316	if (device_user_pasid_enabled(idxd) && !IS_ERR_OR_NULL(sva))
317		iommu_sva_unbind_device(sva);
318failed:
319	mutex_unlock(&wq->wq_lock);
320	kfree(ctx);
321	return rc;
322}
323
324static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid)
325{
326	struct idxd_device *idxd = wq->idxd;
327	struct idxd_evl *evl = idxd->evl;
328	union evl_status_reg status;
329	u16 h, t, size;
330	int ent_size = evl_ent_size(idxd);
331	struct __evl_entry *entry_head;
332
333	if (!evl)
334		return;
335
336	mutex_lock(&evl->lock);
337	status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET);
338	t = status.tail;
339	h = status.head;
340	size = evl->size;
341
342	while (h != t) {
343		entry_head = (struct __evl_entry *)(evl->log + (h * ent_size));
344		if (entry_head->pasid == pasid && entry_head->wq_idx == wq->id)
345			set_bit(h, evl->bmap);
346		h = (h + 1) % size;
347	}
348	if (wq->wq)
349		drain_workqueue(wq->wq);
350
351	mutex_unlock(&evl->lock);
352}
353
354static int idxd_cdev_release(struct inode *node, struct file *filep)
355{
356	struct idxd_user_context *ctx = filep->private_data;
357	struct idxd_wq *wq = ctx->wq;
358	struct idxd_device *idxd = wq->idxd;
359	struct device *dev = &idxd->pdev->dev;
360
361	dev_dbg(dev, "%s called\n", __func__);
362	filep->private_data = NULL;
363
364	device_unregister(user_ctx_dev(ctx));
365
366	return 0;
367}
368
369static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
370		     const char *func)
371{
372	struct device *dev = &wq->idxd->pdev->dev;
373
374	if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
375		dev_info_ratelimited(dev,
376				     "%s: %s: mapping too large: %lu\n",
377				     current->comm, func,
378				     vma->vm_end - vma->vm_start);
379		return -EINVAL;
380	}
381
382	return 0;
383}
384
385static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
386{
387	struct idxd_user_context *ctx = filp->private_data;
388	struct idxd_wq *wq = ctx->wq;
389	struct idxd_device *idxd = wq->idxd;
390	struct pci_dev *pdev = idxd->pdev;
391	phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
392	unsigned long pfn;
393	int rc;
394
395	dev_dbg(&pdev->dev, "%s called\n", __func__);
396
397	/*
398	 * Due to an erratum in some of the devices supported by the driver,
399	 * direct user submission to the device can be unsafe.
400	 * (See the INTEL-SA-01084 security advisory)
401	 *
402	 * For the devices that exhibit this behavior, require that the user
403	 * has CAP_SYS_RAWIO capabilities.
404	 */
405	if (!idxd->user_submission_safe && !capable(CAP_SYS_RAWIO))
406		return -EPERM;
407
408	if (current->mm != ctx->mm)
409		return -EPERM;
410
411	rc = check_vma(wq, vma, __func__);
412	if (rc < 0)
413		return rc;
414
415	vm_flags_set(vma, VM_DONTCOPY);
416	pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
417				IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
418	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
419	vma->vm_private_data = ctx;
420
421	return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
422			vma->vm_page_prot);
423}
424
425static int idxd_submit_user_descriptor(struct idxd_user_context *ctx,
426				       struct dsa_hw_desc __user *udesc)
427{
428	struct idxd_wq *wq = ctx->wq;
429	struct idxd_dev *idxd_dev = &wq->idxd->idxd_dev;
430	const uint64_t comp_addr_align = is_dsa_dev(idxd_dev) ? 0x20 : 0x40;
431	void __iomem *portal = idxd_wq_portal_addr(wq);
432	struct dsa_hw_desc descriptor __aligned(64);
433	int rc;
434
435	rc = copy_from_user(&descriptor, udesc, sizeof(descriptor));
436	if (rc)
437		return -EFAULT;
438
439	/*
440	 * DSA devices are capable of indirect ("batch") command submission.
441	 * On devices where direct user submissions are not safe, we cannot
442	 * allow this since there is no good way for us to verify these
443	 * indirect commands. Narrow the restriction of operations with the
444	 * BATCH opcode to only DSA version 1 devices.
445	 */
446	if (is_dsa_dev(idxd_dev) && descriptor.opcode == DSA_OPCODE_BATCH &&
447	    wq->idxd->hw.version == DEVICE_VERSION_1 &&
448	    !wq->idxd->user_submission_safe)
449		return -EINVAL;
450	/*
451	 * As per the programming specification, the completion address must be
452	 * aligned to 32 or 64 bytes. If this is violated the hardware
453	 * engine can get very confused (security issue).
454	 */
455	if (!IS_ALIGNED(descriptor.completion_addr, comp_addr_align))
456		return -EINVAL;
457
458	if (wq_dedicated(wq))
459		iosubmit_cmds512(portal, &descriptor, 1);
460	else {
461		descriptor.priv = 0;
462		descriptor.pasid = ctx->pasid;
463		rc = idxd_enqcmds(wq, portal, &descriptor);
464		if (rc < 0)
465			return rc;
466	}
467
468	return 0;
469}
470
471static ssize_t idxd_cdev_write(struct file *filp, const char __user *buf, size_t len,
472			       loff_t *unused)
473{
474	struct dsa_hw_desc __user *udesc = (struct dsa_hw_desc __user *)buf;
475	struct idxd_user_context *ctx = filp->private_data;
476	ssize_t written = 0;
477	int i;
478
479	if (current->mm != ctx->mm)
480		return -EPERM;
481
482	for (i = 0; i < len/sizeof(struct dsa_hw_desc); i++) {
483		int rc = idxd_submit_user_descriptor(ctx, udesc + i);
484
485		if (rc)
486			return written ? written : rc;
487
488		written += sizeof(struct dsa_hw_desc);
489	}
490
491	return written;
492}
493
494static __poll_t idxd_cdev_poll(struct file *filp,
495			       struct poll_table_struct *wait)
496{
497	struct idxd_user_context *ctx = filp->private_data;
498	struct idxd_wq *wq = ctx->wq;
499	struct idxd_device *idxd = wq->idxd;
500	__poll_t out = 0;
501
502	if (current->mm != ctx->mm)
503		return POLLNVAL;
504
505	poll_wait(filp, &wq->err_queue, wait);
506	spin_lock(&idxd->dev_lock);
507	if (idxd->sw_err.valid)
508		out = EPOLLIN | EPOLLRDNORM;
509	spin_unlock(&idxd->dev_lock);
510
511	return out;
512}
513
514static const struct file_operations idxd_cdev_fops = {
515	.owner = THIS_MODULE,
516	.open = idxd_cdev_open,
517	.release = idxd_cdev_release,
518	.mmap = idxd_cdev_mmap,
519	.write = idxd_cdev_write,
520	.poll = idxd_cdev_poll,
521};
522
523int idxd_cdev_get_major(struct idxd_device *idxd)
524{
525	return MAJOR(ictx[idxd->data->type].devt);
526}
527
528int idxd_wq_add_cdev(struct idxd_wq *wq)
529{
530	struct idxd_device *idxd = wq->idxd;
531	struct idxd_cdev *idxd_cdev;
532	struct cdev *cdev;
533	struct device *dev;
534	struct idxd_cdev_context *cdev_ctx;
535	int rc, minor;
536
537	idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL);
538	if (!idxd_cdev)
539		return -ENOMEM;
540
541	idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV;
542	idxd_cdev->wq = wq;
543	cdev = &idxd_cdev->cdev;
544	dev = cdev_dev(idxd_cdev);
545	cdev_ctx = &ictx[wq->idxd->data->type];
546	minor = ida_alloc_max(&cdev_ctx->minor_ida, MINORMASK, GFP_KERNEL);
547	if (minor < 0) {
548		kfree(idxd_cdev);
549		return minor;
550	}
551	idxd_cdev->minor = minor;
552
553	device_initialize(dev);
554	dev->parent = wq_confdev(wq);
555	dev->bus = &dsa_bus_type;
556	dev->type = &idxd_cdev_device_type;
557	dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
558
559	rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id);
560	if (rc < 0)
561		goto err;
562
563	wq->idxd_cdev = idxd_cdev;
564	cdev_init(cdev, &idxd_cdev_fops);
565	rc = cdev_device_add(cdev, dev);
566	if (rc) {
567		dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
568		goto err;
569	}
570
571	return 0;
572
573 err:
574	put_device(dev);
575	wq->idxd_cdev = NULL;
576	return rc;
577}
578
579void idxd_wq_del_cdev(struct idxd_wq *wq)
580{
581	struct idxd_cdev_context *cdev_ctx;
582	struct idxd_cdev *idxd_cdev;
583
584	idxd_cdev = wq->idxd_cdev;
585	wq->idxd_cdev = NULL;
586	cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev));
587
588	cdev_ctx = &ictx[wq->idxd->data->type];
589	ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor);
590	put_device(cdev_dev(idxd_cdev));
591}
592
593static int idxd_user_drv_probe(struct idxd_dev *idxd_dev)
594{
595	struct device *dev = &idxd_dev->conf_dev;
596	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
597	struct idxd_device *idxd = wq->idxd;
598	int rc;
599
600	if (idxd->state != IDXD_DEV_ENABLED)
601		return -ENXIO;
602
603	mutex_lock(&wq->wq_lock);
604
605	if (!idxd_wq_driver_name_match(wq, dev)) {
606		idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME;
607		rc = -ENODEV;
608		goto wq_err;
609	}
610
611	/*
612	 * User type WQ is enabled only when SVA is enabled for two reasons:
613	 *   - If no IOMMU or IOMMU Passthrough without SVA, userspace
614	 *     can directly access physical address through the WQ.
615	 *   - The IDXD cdev driver does not provide any ways to pin
616	 *     user pages and translate the address from user VA to IOVA or
617	 *     PA without IOMMU SVA. Therefore the application has no way
618	 *     to instruct the device to perform DMA function. This makes
619	 *     the cdev not usable for normal application usage.
620	 */
621	if (!device_user_pasid_enabled(idxd)) {
622		idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU;
623		dev_dbg(&idxd->pdev->dev,
624			"User type WQ cannot be enabled without SVA.\n");
625
626		rc = -EOPNOTSUPP;
627		goto wq_err;
628	}
629
630	wq->wq = create_workqueue(dev_name(wq_confdev(wq)));
631	if (!wq->wq) {
632		rc = -ENOMEM;
633		goto wq_err;
634	}
635
636	wq->type = IDXD_WQT_USER;
637	rc = idxd_drv_enable_wq(wq);
638	if (rc < 0)
639		goto err;
640
641	rc = idxd_wq_add_cdev(wq);
642	if (rc < 0) {
643		idxd->cmd_status = IDXD_SCMD_CDEV_ERR;
644		goto err_cdev;
645	}
646
647	idxd->cmd_status = 0;
648	mutex_unlock(&wq->wq_lock);
649	return 0;
650
651err_cdev:
652	idxd_drv_disable_wq(wq);
653err:
654	destroy_workqueue(wq->wq);
655	wq->type = IDXD_WQT_NONE;
656wq_err:
657	mutex_unlock(&wq->wq_lock);
658	return rc;
659}
660
661static void idxd_user_drv_remove(struct idxd_dev *idxd_dev)
662{
663	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
664
665	mutex_lock(&wq->wq_lock);
666	idxd_wq_del_cdev(wq);
667	idxd_drv_disable_wq(wq);
668	wq->type = IDXD_WQT_NONE;
669	destroy_workqueue(wq->wq);
670	wq->wq = NULL;
671	mutex_unlock(&wq->wq_lock);
672}
673
674static enum idxd_dev_type dev_types[] = {
675	IDXD_DEV_WQ,
676	IDXD_DEV_NONE,
677};
678
679struct idxd_device_driver idxd_user_drv = {
680	.probe = idxd_user_drv_probe,
681	.remove = idxd_user_drv_remove,
682	.name = "user",
683	.type = dev_types,
684};
685EXPORT_SYMBOL_GPL(idxd_user_drv);
686
687int idxd_cdev_register(void)
688{
689	int rc, i;
690
691	for (i = 0; i < IDXD_TYPE_MAX; i++) {
692		ida_init(&ictx[i].minor_ida);
693		rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
694					 ictx[i].name);
695		if (rc)
696			goto err_free_chrdev_region;
697	}
698
699	return 0;
700
701err_free_chrdev_region:
702	for (i--; i >= 0; i--)
703		unregister_chrdev_region(ictx[i].devt, MINORMASK);
704
705	return rc;
706}
707
708void idxd_cdev_remove(void)
709{
710	int i;
711
712	for (i = 0; i < IDXD_TYPE_MAX; i++) {
713		unregister_chrdev_region(ictx[i].devt, MINORMASK);
714		ida_destroy(&ictx[i].minor_ida);
715	}
716}
717
718/**
719 * idxd_copy_cr - copy completion record to user address space found by wq and
720 *		  PASID
721 * @wq:		work queue
722 * @pasid:	PASID
723 * @addr:	user fault address to write
724 * @cr:		completion record
725 * @len:	number of bytes to copy
726 *
727 * This is called by a work that handles completion record fault.
728 *
729 * Return: number of bytes copied.
730 */
731int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr,
732		 void *cr, int len)
733{
734	struct device *dev = &wq->idxd->pdev->dev;
735	int left = len, status_size = 1;
736	struct idxd_user_context *ctx;
737	struct mm_struct *mm;
738
739	mutex_lock(&wq->uc_lock);
740
741	ctx = xa_load(&wq->upasid_xa, pasid);
742	if (!ctx) {
743		dev_warn(dev, "No user context\n");
744		goto out;
745	}
746
747	mm = ctx->mm;
748	/*
749	 * The completion record fault handling work is running in kernel
750	 * thread context. It temporarily switches to the mm to copy cr
751	 * to addr in the mm.
752	 */
753	kthread_use_mm(mm);
754	left = copy_to_user((void __user *)addr + status_size, cr + status_size,
755			    len - status_size);
756	/*
757	 * Copy status only after the rest of completion record is copied
758	 * successfully so that the user gets the complete completion record
759	 * when a non-zero status is polled.
760	 */
761	if (!left) {
762		u8 status;
763
764		/*
765		 * Ensure that the completion record's status field is written
766		 * after the rest of the completion record has been written.
767		 * This ensures that the user receives the correct completion
768		 * record information once polling for a non-zero status.
769		 */
770		wmb();
771		status = *(u8 *)cr;
772		if (put_user(status, (u8 __user *)addr))
773			left += status_size;
774	} else {
775		left += status_size;
776	}
777	kthread_unuse_mm(mm);
778
779out:
780	mutex_unlock(&wq->uc_lock);
781
782	return len - left;
783}