mr.c

  1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
  2/*
  3 * Copyright(c) 2016 Intel Corporation.
  4 */
  5
  6#include <linux/slab.h>
  7#include <linux/vmalloc.h>
  8#include <rdma/ib_umem.h>
  9#include <rdma/rdma_vt.h>
 10#include "vt.h"
 11#include "mr.h"
 12#include "trace.h"
 13
 14/**
 15 * rvt_driver_mr_init - Init MR resources per driver
 16 * @rdi: rvt dev struct
 17 *
 18 * Do any intilization needed when a driver registers with rdmavt.
 19 *
 20 * Return: 0 on success or errno on failure
 21 */
 22int rvt_driver_mr_init(struct rvt_dev_info *rdi)
 23{
 24	unsigned int lkey_table_size = rdi->dparms.lkey_table_size;
 25	unsigned lk_tab_size;
 26	int i;
 27
 28	/*
 29	 * The top hfi1_lkey_table_size bits are used to index the
 30	 * table.  The lower 8 bits can be owned by the user (copied from
 31	 * the LKEY).  The remaining bits act as a generation number or tag.
 32	 */
 33	if (!lkey_table_size)
 34		return -EINVAL;
 35
 36	spin_lock_init(&rdi->lkey_table.lock);
 37
 38	/* ensure generation is at least 4 bits */
 39	if (lkey_table_size > RVT_MAX_LKEY_TABLE_BITS) {
 40		rvt_pr_warn(rdi, "lkey bits %u too large, reduced to %u\n",
 41			    lkey_table_size, RVT_MAX_LKEY_TABLE_BITS);
 42		rdi->dparms.lkey_table_size = RVT_MAX_LKEY_TABLE_BITS;
 43		lkey_table_size = rdi->dparms.lkey_table_size;
 44	}
 45	rdi->lkey_table.max = 1 << lkey_table_size;
 46	rdi->lkey_table.shift = 32 - lkey_table_size;
 47	lk_tab_size = rdi->lkey_table.max * sizeof(*rdi->lkey_table.table);
 48	rdi->lkey_table.table = (struct rvt_mregion __rcu **)
 49			       vmalloc_node(lk_tab_size, rdi->dparms.node);
 50	if (!rdi->lkey_table.table)
 51		return -ENOMEM;
 52
 53	RCU_INIT_POINTER(rdi->dma_mr, NULL);
 54	for (i = 0; i < rdi->lkey_table.max; i++)
 55		RCU_INIT_POINTER(rdi->lkey_table.table[i], NULL);
 56
 57	rdi->dparms.props.max_mr = rdi->lkey_table.max;
 58	return 0;
 59}
 60
 61/**
 62 * rvt_mr_exit - clean up MR
 63 * @rdi: rvt dev structure
 64 *
 65 * called when drivers have unregistered or perhaps failed to register with us
 66 */
 67void rvt_mr_exit(struct rvt_dev_info *rdi)
 68{
 69	if (rdi->dma_mr)
 70		rvt_pr_err(rdi, "DMA MR not null!\n");
 71
 72	vfree(rdi->lkey_table.table);
 73}
 74
 75static void rvt_deinit_mregion(struct rvt_mregion *mr)
 76{
 77	int i = mr->mapsz;
 78
 79	mr->mapsz = 0;
 80	while (i)
 81		kfree(mr->map[--i]);
 82	percpu_ref_exit(&mr->refcount);
 83}
 84
 85static void __rvt_mregion_complete(struct percpu_ref *ref)
 86{
 87	struct rvt_mregion *mr = container_of(ref, struct rvt_mregion,
 88					      refcount);
 89
 90	complete(&mr->comp);
 91}
 92
 93static int rvt_init_mregion(struct rvt_mregion *mr, struct ib_pd *pd,
 94			    int count, unsigned int percpu_flags)
 95{
 96	int m, i = 0;
 97	struct rvt_dev_info *dev = ib_to_rvt(pd->device);
 98
 99	mr->mapsz = 0;
100	m = (count + RVT_SEGSZ - 1) / RVT_SEGSZ;
101	for (; i < m; i++) {
102		mr->map[i] = kzalloc_node(sizeof(*mr->map[0]), GFP_KERNEL,
103					  dev->dparms.node);
104		if (!mr->map[i])
105			goto bail;
106		mr->mapsz++;
107	}
108	init_completion(&mr->comp);
109	/* count returning the ptr to user */
110	if (percpu_ref_init(&mr->refcount, &__rvt_mregion_complete,
111			    percpu_flags, GFP_KERNEL))
112		goto bail;
113
114	atomic_set(&mr->lkey_invalid, 0);
115	mr->pd = pd;
116	mr->max_segs = count;
117	return 0;
118bail:
119	rvt_deinit_mregion(mr);
120	return -ENOMEM;
121}
122
123/**
124 * rvt_alloc_lkey - allocate an lkey
125 * @mr: memory region that this lkey protects
126 * @dma_region: 0->normal key, 1->restricted DMA key
127 *
128 * Returns 0 if successful, otherwise returns -errno.
129 *
130 * Increments mr reference count as required.
131 *
132 * Sets the lkey field mr for non-dma regions.
133 *
134 */
135static int rvt_alloc_lkey(struct rvt_mregion *mr, int dma_region)
136{
137	unsigned long flags;
138	u32 r;
139	u32 n;
140	int ret = 0;
141	struct rvt_dev_info *dev = ib_to_rvt(mr->pd->device);
142	struct rvt_lkey_table *rkt = &dev->lkey_table;
143
144	rvt_get_mr(mr);
145	spin_lock_irqsave(&rkt->lock, flags);
146
147	/* special case for dma_mr lkey == 0 */
148	if (dma_region) {
149		struct rvt_mregion *tmr;
150
151		tmr = rcu_access_pointer(dev->dma_mr);
152		if (!tmr) {
153			mr->lkey_published = 1;
154			/* Insure published written first */
155			rcu_assign_pointer(dev->dma_mr, mr);
156			rvt_get_mr(mr);
157		}
158		goto success;
159	}
160
161	/* Find the next available LKEY */
162	r = rkt->next;
163	n = r;
164	for (;;) {
165		if (!rcu_access_pointer(rkt->table[r]))
166			break;
167		r = (r + 1) & (rkt->max - 1);
168		if (r == n)
169			goto bail;
170	}
171	rkt->next = (r + 1) & (rkt->max - 1);
172	/*
173	 * Make sure lkey is never zero which is reserved to indicate an
174	 * unrestricted LKEY.
175	 */
176	rkt->gen++;
177	/*
178	 * bits are capped to ensure enough bits for generation number
179	 */
180	mr->lkey = (r << (32 - dev->dparms.lkey_table_size)) |
181		((((1 << (24 - dev->dparms.lkey_table_size)) - 1) & rkt->gen)
182		 << 8);
183	if (mr->lkey == 0) {
184		mr->lkey |= 1 << 8;
185		rkt->gen++;
186	}
187	mr->lkey_published = 1;
188	/* Insure published written first */
189	rcu_assign_pointer(rkt->table[r], mr);
190success:
191	spin_unlock_irqrestore(&rkt->lock, flags);
192out:
193	return ret;
194bail:
195	rvt_put_mr(mr);
196	spin_unlock_irqrestore(&rkt->lock, flags);
197	ret = -ENOMEM;
198	goto out;
199}
200
201/**
202 * rvt_free_lkey - free an lkey
203 * @mr: mr to free from tables
204 */
205static void rvt_free_lkey(struct rvt_mregion *mr)
206{
207	unsigned long flags;
208	u32 lkey = mr->lkey;
209	u32 r;
210	struct rvt_dev_info *dev = ib_to_rvt(mr->pd->device);
211	struct rvt_lkey_table *rkt = &dev->lkey_table;
212	int freed = 0;
213
214	spin_lock_irqsave(&rkt->lock, flags);
215	if (!lkey) {
216		if (mr->lkey_published) {
217			mr->lkey_published = 0;
218			/* insure published is written before pointer */
219			rcu_assign_pointer(dev->dma_mr, NULL);
220			rvt_put_mr(mr);
221		}
222	} else {
223		if (!mr->lkey_published)
224			goto out;
225		r = lkey >> (32 - dev->dparms.lkey_table_size);
226		mr->lkey_published = 0;
227		/* insure published is written before pointer */
228		rcu_assign_pointer(rkt->table[r], NULL);
229	}
230	freed++;
231out:
232	spin_unlock_irqrestore(&rkt->lock, flags);
233	if (freed)
234		percpu_ref_kill(&mr->refcount);
235}
236
237static struct rvt_mr *__rvt_alloc_mr(int count, struct ib_pd *pd)
238{
239	struct rvt_mr *mr;
240	int rval = -ENOMEM;
241	int m;
242
243	/* Allocate struct plus pointers to first level page tables. */
244	m = (count + RVT_SEGSZ - 1) / RVT_SEGSZ;
245	mr = kzalloc(struct_size(mr, mr.map, m), GFP_KERNEL);
246	if (!mr)
247		goto bail;
248
249	rval = rvt_init_mregion(&mr->mr, pd, count, 0);
250	if (rval)
251		goto bail;
252	/*
253	 * ib_reg_phys_mr() will initialize mr->ibmr except for
254	 * lkey and rkey.
255	 */
256	rval = rvt_alloc_lkey(&mr->mr, 0);
257	if (rval)
258		goto bail_mregion;
259	mr->ibmr.lkey = mr->mr.lkey;
260	mr->ibmr.rkey = mr->mr.lkey;
261done:
262	return mr;
263
264bail_mregion:
265	rvt_deinit_mregion(&mr->mr);
266bail:
267	kfree(mr);
268	mr = ERR_PTR(rval);
269	goto done;
270}
271
272static void __rvt_free_mr(struct rvt_mr *mr)
273{
274	rvt_free_lkey(&mr->mr);
275	rvt_deinit_mregion(&mr->mr);
276	kfree(mr);
277}
278
279/**
280 * rvt_get_dma_mr - get a DMA memory region
281 * @pd: protection domain for this memory region
282 * @acc: access flags
283 *
284 * Return: the memory region on success, otherwise returns an errno.
285 */
286struct ib_mr *rvt_get_dma_mr(struct ib_pd *pd, int acc)
287{
288	struct rvt_mr *mr;
289	struct ib_mr *ret;
290	int rval;
291
292	if (ibpd_to_rvtpd(pd)->user)
293		return ERR_PTR(-EPERM);
294
295	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
296	if (!mr) {
297		ret = ERR_PTR(-ENOMEM);
298		goto bail;
299	}
300
301	rval = rvt_init_mregion(&mr->mr, pd, 0, 0);
302	if (rval) {
303		ret = ERR_PTR(rval);
304		goto bail;
305	}
306
307	rval = rvt_alloc_lkey(&mr->mr, 1);
308	if (rval) {
309		ret = ERR_PTR(rval);
310		goto bail_mregion;
311	}
312
313	mr->mr.access_flags = acc;
314	ret = &mr->ibmr;
315done:
316	return ret;
317
318bail_mregion:
319	rvt_deinit_mregion(&mr->mr);
320bail:
321	kfree(mr);
322	goto done;
323}
324
325/**
326 * rvt_reg_user_mr - register a userspace memory region
327 * @pd: protection domain for this memory region
328 * @start: starting userspace address
329 * @length: length of region to register
330 * @virt_addr: associated virtual address
331 * @mr_access_flags: access flags for this memory region
332 * @dmah: dma handle
333 * @udata: unused by the driver
334 *
335 * Return: the memory region on success, otherwise returns an errno.
336 */
337struct ib_mr *rvt_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
338			      u64 virt_addr, int mr_access_flags,
339			      struct ib_dmah *dmah,
340			      struct ib_udata *udata)
341{
342	struct rvt_mr *mr;
343	struct ib_umem *umem;
344	struct sg_page_iter sg_iter;
345	int n, m;
346	struct ib_mr *ret;
347
348	if (dmah)
349		return ERR_PTR(-EOPNOTSUPP);
350
351	if (length == 0)
352		return ERR_PTR(-EINVAL);
353
354	umem = ib_umem_get(pd->device, start, length, mr_access_flags);
355	if (IS_ERR(umem))
356		return ERR_CAST(umem);
357
358	n = ib_umem_num_pages(umem);
359
360	mr = __rvt_alloc_mr(n, pd);
361	if (IS_ERR(mr)) {
362		ret = ERR_CAST(mr);
363		goto bail_umem;
364	}
365
366	mr->mr.user_base = start;
367	mr->mr.iova = virt_addr;
368	mr->mr.length = length;
369	mr->mr.offset = ib_umem_offset(umem);
370	mr->mr.access_flags = mr_access_flags;
371	mr->umem = umem;
372
373	mr->mr.page_shift = PAGE_SHIFT;
374	m = 0;
375	n = 0;
376	for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) {
377		void *vaddr;
378
379		vaddr = page_address(sg_page_iter_page(&sg_iter));
380		if (!vaddr) {
381			ret = ERR_PTR(-EINVAL);
382			goto bail_inval;
383		}
384		mr->mr.map[m]->segs[n].vaddr = vaddr;
385		mr->mr.map[m]->segs[n].length = PAGE_SIZE;
386		trace_rvt_mr_user_seg(&mr->mr, m, n, vaddr, PAGE_SIZE);
387		if (++n == RVT_SEGSZ) {
388			m++;
389			n = 0;
390		}
391	}
392	return &mr->ibmr;
393
394bail_inval:
395	__rvt_free_mr(mr);
396
397bail_umem:
398	ib_umem_release(umem);
399
400	return ret;
401}
402
403/**
404 * rvt_dereg_clean_qp_cb - callback from iterator
405 * @qp: the qp
406 * @v: the mregion (as u64)
407 *
408 * This routine fields the callback for all QPs and
409 * for QPs in the same PD as the MR will call the
410 * rvt_qp_mr_clean() to potentially cleanup references.
411 */
412static void rvt_dereg_clean_qp_cb(struct rvt_qp *qp, u64 v)
413{
414	struct rvt_mregion *mr = (struct rvt_mregion *)v;
415
416	/* skip PDs that are not ours */
417	if (mr->pd != qp->ibqp.pd)
418		return;
419	rvt_qp_mr_clean(qp, mr->lkey);
420}
421
422/**
423 * rvt_dereg_clean_qps - find QPs for reference cleanup
424 * @mr: the MR that is being deregistered
425 *
426 * This routine iterates RC QPs looking for references
427 * to the lkey noted in mr.
428 */
429static void rvt_dereg_clean_qps(struct rvt_mregion *mr)
430{
431	struct rvt_dev_info *rdi = ib_to_rvt(mr->pd->device);
432
433	rvt_qp_iter(rdi, (u64)mr, rvt_dereg_clean_qp_cb);
434}
435
436/**
437 * rvt_check_refs - check references
438 * @mr: the megion
439 * @t: the caller identification
440 *
441 * This routine checks MRs holding a reference during
442 * when being de-registered.
443 *
444 * If the count is non-zero, the code calls a clean routine then
445 * waits for the timeout for the count to zero.
446 */
447static int rvt_check_refs(struct rvt_mregion *mr, const char *t)
448{
449	unsigned long timeout;
450	struct rvt_dev_info *rdi = ib_to_rvt(mr->pd->device);
451
452	if (mr->lkey) {
453		/* avoid dma mr */
454		rvt_dereg_clean_qps(mr);
455		/* @mr was indexed on rcu protected @lkey_table */
456		synchronize_rcu();
457	}
458
459	timeout = wait_for_completion_timeout(&mr->comp, 5 * HZ);
460	if (!timeout) {
461		rvt_pr_err(rdi,
462			   "%s timeout mr %p pd %p lkey %x refcount %ld\n",
463			   t, mr, mr->pd, mr->lkey,
464			   atomic_long_read(&mr->refcount.data->count));
465		rvt_get_mr(mr);
466		return -EBUSY;
467	}
468	return 0;
469}
470
471/**
472 * rvt_mr_has_lkey - is MR
473 * @mr: the mregion
474 * @lkey: the lkey
475 */
476bool rvt_mr_has_lkey(struct rvt_mregion *mr, u32 lkey)
477{
478	return mr && lkey == mr->lkey;
479}
480
481/**
482 * rvt_ss_has_lkey - is mr in sge tests
483 * @ss: the sge state
484 * @lkey: the lkey
485 *
486 * This code tests for an MR in the indicated
487 * sge state.
488 */
489bool rvt_ss_has_lkey(struct rvt_sge_state *ss, u32 lkey)
490{
491	int i;
492	bool rval = false;
493
494	if (!ss->num_sge)
495		return rval;
496	/* first one */
497	rval = rvt_mr_has_lkey(ss->sge.mr, lkey);
498	/* any others */
499	for (i = 0; !rval && i < ss->num_sge - 1; i++)
500		rval = rvt_mr_has_lkey(ss->sg_list[i].mr, lkey);
501	return rval;
502}
503
504/**
505 * rvt_dereg_mr - unregister and free a memory region
506 * @ibmr: the memory region to free
507 * @udata: unused by the driver
508 *
509 * Note that this is called to free MRs created by rvt_get_dma_mr()
510 * or rvt_reg_user_mr().
511 *
512 * Returns 0 on success.
513 */
514int rvt_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
515{
516	struct rvt_mr *mr = to_imr(ibmr);
517	int ret;
518
519	rvt_free_lkey(&mr->mr);
520
521	rvt_put_mr(&mr->mr); /* will set completion if last */
522	ret = rvt_check_refs(&mr->mr, __func__);
523	if (ret)
524		goto out;
525	rvt_deinit_mregion(&mr->mr);
526	ib_umem_release(mr->umem);
527	kfree(mr);
528out:
529	return ret;
530}
531
532/**
533 * rvt_alloc_mr - Allocate a memory region usable with the
534 * @pd: protection domain for this memory region
535 * @mr_type: mem region type
536 * @max_num_sg: Max number of segments allowed
537 *
538 * Return: the memory region on success, otherwise return an errno.
539 */
540struct ib_mr *rvt_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
541			   u32 max_num_sg)
542{
543	struct rvt_mr *mr;
544
545	if (mr_type != IB_MR_TYPE_MEM_REG)
546		return ERR_PTR(-EINVAL);
547
548	mr = __rvt_alloc_mr(max_num_sg, pd);
549	if (IS_ERR(mr))
550		return ERR_CAST(mr);
551
552	return &mr->ibmr;
553}
554
555/**
556 * rvt_set_page - page assignment function called by ib_sg_to_pages
557 * @ibmr: memory region
558 * @addr: dma address of mapped page
559 *
560 * Return: 0 on success
561 */
562static int rvt_set_page(struct ib_mr *ibmr, u64 addr)
563{
564	struct rvt_mr *mr = to_imr(ibmr);
565	u32 ps = 1 << mr->mr.page_shift;
566	u32 mapped_segs = mr->mr.length >> mr->mr.page_shift;
567	int m, n;
568
569	if (unlikely(mapped_segs == mr->mr.max_segs))
570		return -ENOMEM;
571
572	m = mapped_segs / RVT_SEGSZ;
573	n = mapped_segs % RVT_SEGSZ;
574	mr->mr.map[m]->segs[n].vaddr = (void *)addr;
575	mr->mr.map[m]->segs[n].length = ps;
576	mr->mr.length += ps;
577	trace_rvt_mr_page_seg(&mr->mr, m, n, (void *)addr, ps);
578
579	return 0;
580}
581
582/**
583 * rvt_map_mr_sg - map sg list and set it the memory region
584 * @ibmr: memory region
585 * @sg: dma mapped scatterlist
586 * @sg_nents: number of entries in sg
587 * @sg_offset: offset in bytes into sg
588 *
589 * Overwrite rvt_mr length with mr length calculated by ib_sg_to_pages.
590 *
591 * Return: number of sg elements mapped to the memory region
592 */
593int rvt_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
594		  int sg_nents, unsigned int *sg_offset)
595{
596	struct rvt_mr *mr = to_imr(ibmr);
597	int ret;
598
599	mr->mr.length = 0;
600	mr->mr.page_shift = PAGE_SHIFT;
601	ret = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rvt_set_page);
602	mr->mr.user_base = ibmr->iova;
603	mr->mr.iova = ibmr->iova;
604	mr->mr.offset = ibmr->iova - (u64)mr->mr.map[0]->segs[0].vaddr;
605	mr->mr.length = (size_t)ibmr->length;
606	trace_rvt_map_mr_sg(ibmr, sg_nents, sg_offset);
607	return ret;
608}
609
610/**
611 * rvt_fast_reg_mr - fast register physical MR
612 * @qp: the queue pair where the work request comes from
613 * @ibmr: the memory region to be registered
614 * @key: updated key for this memory region
615 * @access: access flags for this memory region
616 *
617 * Returns 0 on success.
618 */
619int rvt_fast_reg_mr(struct rvt_qp *qp, struct ib_mr *ibmr, u32 key,
620		    int access)
621{
622	struct rvt_mr *mr = to_imr(ibmr);
623
624	if (qp->ibqp.pd != mr->mr.pd)
625		return -EACCES;
626
627	/* not applicable to dma MR or user MR */
628	if (!mr->mr.lkey || mr->umem)
629		return -EINVAL;
630
631	if ((key & 0xFFFFFF00) != (mr->mr.lkey & 0xFFFFFF00))
632		return -EINVAL;
633
634	ibmr->lkey = key;
635	ibmr->rkey = key;
636	mr->mr.lkey = key;
637	mr->mr.access_flags = access;
638	mr->mr.iova = ibmr->iova;
639	atomic_set(&mr->mr.lkey_invalid, 0);
640
641	return 0;
642}
643EXPORT_SYMBOL(rvt_fast_reg_mr);
644
645/**
646 * rvt_invalidate_rkey - invalidate an MR rkey
647 * @qp: queue pair associated with the invalidate op
648 * @rkey: rkey to invalidate
649 *
650 * Returns 0 on success.
651 */
652int rvt_invalidate_rkey(struct rvt_qp *qp, u32 rkey)
653{
654	struct rvt_dev_info *dev = ib_to_rvt(qp->ibqp.device);
655	struct rvt_lkey_table *rkt = &dev->lkey_table;
656	struct rvt_mregion *mr;
657
658	if (rkey == 0)
659		return -EINVAL;
660
661	rcu_read_lock();
662	mr = rcu_dereference(
663		rkt->table[(rkey >> (32 - dev->dparms.lkey_table_size))]);
664	if (unlikely(!mr || mr->lkey != rkey || qp->ibqp.pd != mr->pd))
665		goto bail;
666
667	atomic_set(&mr->lkey_invalid, 1);
668	rcu_read_unlock();
669	return 0;
670
671bail:
672	rcu_read_unlock();
673	return -EINVAL;
674}
675EXPORT_SYMBOL(rvt_invalidate_rkey);
676
677/**
678 * rvt_sge_adjacent - is isge compressible
679 * @last_sge: last outgoing SGE written
680 * @sge: SGE to check
681 *
682 * If adjacent will update last_sge to add length.
683 *
684 * Return: true if isge is adjacent to last sge
685 */
686static inline bool rvt_sge_adjacent(struct rvt_sge *last_sge,
687				    struct ib_sge *sge)
688{
689	if (last_sge && sge->lkey == last_sge->mr->lkey &&
690	    ((uint64_t)(last_sge->vaddr + last_sge->length) == sge->addr)) {
691		if (sge->lkey) {
692			if (unlikely((sge->addr - last_sge->mr->user_base +
693			      sge->length > last_sge->mr->length)))
694				return false; /* overrun, caller will catch */
695		} else {
696			last_sge->length += sge->length;
697		}
698		last_sge->sge_length += sge->length;
699		trace_rvt_sge_adjacent(last_sge, sge);
700		return true;
701	}
702	return false;
703}
704
705/**
706 * rvt_lkey_ok - check IB SGE for validity and initialize
707 * @rkt: table containing lkey to check SGE against
708 * @pd: protection domain
709 * @isge: outgoing internal SGE
710 * @last_sge: last outgoing SGE written
711 * @sge: SGE to check
712 * @acc: access flags
713 *
714 * Check the IB SGE for validity and initialize our internal version
715 * of it.
716 *
717 * Increments the reference count when a new sge is stored.
718 *
719 * Return: 0 if compressed, 1 if added , otherwise returns -errno.
720 */
721int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
722		struct rvt_sge *isge, struct rvt_sge *last_sge,
723		struct ib_sge *sge, int acc)
724{
725	struct rvt_mregion *mr;
726	unsigned n, m;
727	size_t off;
728
729	/*
730	 * We use LKEY == zero for kernel virtual addresses
731	 * (see rvt_get_dma_mr()).
732	 */
733	if (sge->lkey == 0) {
734		struct rvt_dev_info *dev = ib_to_rvt(pd->ibpd.device);
735
736		if (pd->user)
737			return -EINVAL;
738		if (rvt_sge_adjacent(last_sge, sge))
739			return 0;
740		rcu_read_lock();
741		mr = rcu_dereference(dev->dma_mr);
742		if (!mr)
743			goto bail;
744		rvt_get_mr(mr);
745		rcu_read_unlock();
746
747		isge->mr = mr;
748		isge->vaddr = (void *)sge->addr;
749		isge->length = sge->length;
750		isge->sge_length = sge->length;
751		isge->m = 0;
752		isge->n = 0;
753		goto ok;
754	}
755	if (rvt_sge_adjacent(last_sge, sge))
756		return 0;
757	rcu_read_lock();
758	mr = rcu_dereference(rkt->table[sge->lkey >> rkt->shift]);
759	if (!mr)
760		goto bail;
761	rvt_get_mr(mr);
762	if (!READ_ONCE(mr->lkey_published))
763		goto bail_unref;
764
765	if (unlikely(atomic_read(&mr->lkey_invalid) ||
766		     mr->lkey != sge->lkey || mr->pd != &pd->ibpd))
767		goto bail_unref;
768
769	off = sge->addr - mr->user_base;
770	if (unlikely(sge->addr < mr->user_base ||
771		     off + sge->length > mr->length ||
772		     (mr->access_flags & acc) != acc))
773		goto bail_unref;
774	rcu_read_unlock();
775
776	off += mr->offset;
777	if (mr->page_shift) {
778		/*
779		 * page sizes are uniform power of 2 so no loop is necessary
780		 * entries_spanned_by_off is the number of times the loop below
781		 * would have executed.
782		*/
783		size_t entries_spanned_by_off;
784
785		entries_spanned_by_off = off >> mr->page_shift;
786		off -= (entries_spanned_by_off << mr->page_shift);
787		m = entries_spanned_by_off / RVT_SEGSZ;
788		n = entries_spanned_by_off % RVT_SEGSZ;
789	} else {
790		m = 0;
791		n = 0;
792		while (off >= mr->map[m]->segs[n].length) {
793			off -= mr->map[m]->segs[n].length;
794			n++;
795			if (n >= RVT_SEGSZ) {
796				m++;
797				n = 0;
798			}
799		}
800	}
801	isge->mr = mr;
802	isge->vaddr = mr->map[m]->segs[n].vaddr + off;
803	isge->length = mr->map[m]->segs[n].length - off;
804	isge->sge_length = sge->length;
805	isge->m = m;
806	isge->n = n;
807ok:
808	trace_rvt_sge_new(isge, sge);
809	return 1;
810bail_unref:
811	rvt_put_mr(mr);
812bail:
813	rcu_read_unlock();
814	return -EINVAL;
815}
816EXPORT_SYMBOL(rvt_lkey_ok);
817
818/**
819 * rvt_rkey_ok - check the IB virtual address, length, and RKEY
820 * @qp: qp for validation
821 * @sge: SGE state
822 * @len: length of data
823 * @vaddr: virtual address to place data
824 * @rkey: rkey to check
825 * @acc: access flags
826 *
827 * Return: 1 if successful, otherwise 0.
828 *
829 * increments the reference count upon success
830 */
831int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge,
832		u32 len, u64 vaddr, u32 rkey, int acc)
833{
834	struct rvt_dev_info *dev = ib_to_rvt(qp->ibqp.device);
835	struct rvt_lkey_table *rkt = &dev->lkey_table;
836	struct rvt_mregion *mr;
837	unsigned n, m;
838	size_t off;
839
840	/*
841	 * We use RKEY == zero for kernel virtual addresses
842	 * (see rvt_get_dma_mr()).
843	 */
844	rcu_read_lock();
845	if (rkey == 0) {
846		struct rvt_pd *pd = ibpd_to_rvtpd(qp->ibqp.pd);
847		struct rvt_dev_info *rdi = ib_to_rvt(pd->ibpd.device);
848
849		if (pd->user)
850			goto bail;
851		mr = rcu_dereference(rdi->dma_mr);
852		if (!mr)
853			goto bail;
854		rvt_get_mr(mr);
855		rcu_read_unlock();
856
857		sge->mr = mr;
858		sge->vaddr = (void *)vaddr;
859		sge->length = len;
860		sge->sge_length = len;
861		sge->m = 0;
862		sge->n = 0;
863		goto ok;
864	}
865
866	mr = rcu_dereference(rkt->table[rkey >> rkt->shift]);
867	if (!mr)
868		goto bail;
869	rvt_get_mr(mr);
870	/* insure mr read is before test */
871	if (!READ_ONCE(mr->lkey_published))
872		goto bail_unref;
873	if (unlikely(atomic_read(&mr->lkey_invalid) ||
874		     mr->lkey != rkey || qp->ibqp.pd != mr->pd))
875		goto bail_unref;
876
877	off = vaddr - mr->iova;
878	if (unlikely(vaddr < mr->iova || off + len > mr->length ||
879		     (mr->access_flags & acc) == 0))
880		goto bail_unref;
881	rcu_read_unlock();
882
883	off += mr->offset;
884	if (mr->page_shift) {
885		/*
886		 * page sizes are uniform power of 2 so no loop is necessary
887		 * entries_spanned_by_off is the number of times the loop below
888		 * would have executed.
889		*/
890		size_t entries_spanned_by_off;
891
892		entries_spanned_by_off = off >> mr->page_shift;
893		off -= (entries_spanned_by_off << mr->page_shift);
894		m = entries_spanned_by_off / RVT_SEGSZ;
895		n = entries_spanned_by_off % RVT_SEGSZ;
896	} else {
897		m = 0;
898		n = 0;
899		while (off >= mr->map[m]->segs[n].length) {
900			off -= mr->map[m]->segs[n].length;
901			n++;
902			if (n >= RVT_SEGSZ) {
903				m++;
904				n = 0;
905			}
906		}
907	}
908	sge->mr = mr;
909	sge->vaddr = mr->map[m]->segs[n].vaddr + off;
910	sge->length = mr->map[m]->segs[n].length - off;
911	sge->sge_length = len;
912	sge->m = m;
913	sge->n = n;
914ok:
915	return 1;
916bail_unref:
917	rvt_put_mr(mr);
918bail:
919	rcu_read_unlock();
920	return 0;
921}
922EXPORT_SYMBOL(rvt_rkey_ok);