rxe_mr.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2/*
  3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
  4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
  5 */
  6
  7#include <linux/libnvdimm.h>
  8
  9#include "rxe.h"
 10#include "rxe_loc.h"
 11
 12/* Return a random 8 bit key value that is
 13 * different than the last_key. Set last_key to -1
 14 * if this is the first key for an MR or MW
 15 */
 16u8 rxe_get_next_key(u32 last_key)
 17{
 18	u8 key;
 19
 20	do {
 21		get_random_bytes(&key, 1);
 22	} while (key == last_key);
 23
 24	return key;
 25}
 26
 27int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
 28{
 29	switch (mr->ibmr.type) {
 30	case IB_MR_TYPE_DMA:
 31		return 0;
 32
 33	case IB_MR_TYPE_USER:
 34	case IB_MR_TYPE_MEM_REG:
 35		if (iova < mr->ibmr.iova ||
 36		    iova + length > mr->ibmr.iova + mr->ibmr.length) {
 37			rxe_dbg_mr(mr, "iova/length out of range\n");
 38			return -EINVAL;
 39		}
 40		return 0;
 41
 42	default:
 43		rxe_dbg_mr(mr, "mr type not supported\n");
 44		return -EINVAL;
 45	}
 46}
 47
 48void rxe_mr_init(int access, struct rxe_mr *mr)
 49{
 50	u32 key = mr->elem.index << 8 | rxe_get_next_key(-1);
 51
 52	/* set ibmr->l/rkey and also copy into private l/rkey
 53	 * for user MRs these will always be the same
 54	 * for cases where caller 'owns' the key portion
 55	 * they may be different until REG_MR WQE is executed.
 56	 */
 57	mr->lkey = mr->ibmr.lkey = key;
 58	mr->rkey = mr->ibmr.rkey = key;
 59
 60	mr->access = access;
 61	mr->ibmr.page_size = PAGE_SIZE;
 62	mr->page_mask = PAGE_MASK;
 63	mr->page_shift = PAGE_SHIFT;
 64	mr->state = RXE_MR_STATE_INVALID;
 65}
 66
 67void rxe_mr_init_dma(int access, struct rxe_mr *mr)
 68{
 69	rxe_mr_init(access, mr);
 70
 71	mr->state = RXE_MR_STATE_VALID;
 72	mr->ibmr.type = IB_MR_TYPE_DMA;
 73}
 74
 75/*
 76 * Convert iova to page_info index. The page_info stores pages of size
 77 * PAGE_SIZE, but MRs can have different page sizes. This function
 78 * handles the conversion for all cases:
 79 *
 80 * 1. mr->page_size > PAGE_SIZE:
 81 *    The MR's iova may not be aligned to mr->page_size. We use the
 82 *    aligned base (iova & page_mask) as reference, then calculate
 83 *    which PAGE_SIZE sub-page the iova falls into.
 84 *
 85 * 2. mr->page_size <= PAGE_SIZE:
 86 *    Use simple shift arithmetic since each page_info entry corresponds
 87 *    to one or more MR pages.
 88 */
 89static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova)
 90{
 91	int idx;
 92
 93	if (mr_page_size(mr) > PAGE_SIZE)
 94		idx = (iova - (mr->ibmr.iova & mr->page_mask)) >> PAGE_SHIFT;
 95	else
 96		idx = (iova >> mr->page_shift) -
 97			(mr->ibmr.iova >> mr->page_shift);
 98
 99	WARN_ON(idx >= mr->nbuf);
100	return idx;
101}
102
103/*
104 * Convert iova to offset within the page_info entry.
105 *
106 * For mr_page_size > PAGE_SIZE, the offset is within the system page.
107 * For mr_page_size <= PAGE_SIZE, the offset is within the MR page size.
108 */
109static unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64 iova)
110{
111	if (mr_page_size(mr) > PAGE_SIZE)
112		return iova & (PAGE_SIZE - 1);
113	else
114		return iova & (mr_page_size(mr) - 1);
115}
116
117static bool is_pmem_page(struct page *pg)
118{
119	unsigned long paddr = page_to_phys(pg);
120
121	return REGION_INTERSECTS ==
122	       region_intersects(paddr, PAGE_SIZE, IORESOURCE_MEM,
123				 IORES_DESC_PERSISTENT_MEMORY);
124}
125
126static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt)
127{
128	struct sg_page_iter sg_iter;
129	struct page *page;
130	bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT);
131
132	WARN_ON(mr_page_size(mr) != PAGE_SIZE);
133
134	__sg_page_iter_start(&sg_iter, sgt->sgl, sgt->orig_nents, 0);
135	if (!__sg_page_iter_next(&sg_iter))
136		return 0;
137
138	while (true) {
139		page = sg_page_iter_page(&sg_iter);
140
141		if (persistent && !is_pmem_page(page)) {
142			rxe_dbg_mr(mr, "Page can't be persistent\n");
143			return -EINVAL;
144		}
145
146		mr->page_info[mr->nbuf].page = page;
147		mr->page_info[mr->nbuf].offset = 0;
148		mr->nbuf++;
149
150		if (!__sg_page_iter_next(&sg_iter))
151			break;
152	}
153
154	return 0;
155}
156
157static int __alloc_mr_page_info(struct rxe_mr *mr, int num_pages)
158{
159	mr->page_info = kcalloc(num_pages, sizeof(struct rxe_mr_page),
160				GFP_KERNEL);
161	if (!mr->page_info)
162		return -ENOMEM;
163
164	mr->max_allowed_buf = num_pages;
165	mr->nbuf = 0;
166
167	return 0;
168}
169
170static int alloc_mr_page_info(struct rxe_mr *mr, int num_pages)
171{
172	int ret;
173
174	WARN_ON(mr->num_buf);
175	ret = __alloc_mr_page_info(mr, num_pages);
176	if (ret)
177		return ret;
178
179	mr->num_buf = num_pages;
180
181	return 0;
182}
183
184static void free_mr_page_info(struct rxe_mr *mr)
185{
186	if (!mr->page_info)
187		return;
188
189	kfree(mr->page_info);
190	mr->page_info = NULL;
191}
192
193int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length,
194		     int access, struct rxe_mr *mr)
195{
196	struct ib_umem *umem;
197	int err;
198
199	rxe_mr_init(access, mr);
200
201	umem = ib_umem_get(&rxe->ib_dev, start, length, access);
202	if (IS_ERR(umem)) {
203		rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n",
204			(int)PTR_ERR(umem));
205		return PTR_ERR(umem);
206	}
207
208	err = alloc_mr_page_info(mr, ib_umem_num_pages(umem));
209	if (err)
210		goto err2;
211
212	err = rxe_mr_fill_pages_from_sgt(mr, &umem->sgt_append.sgt);
213	if (err)
214		goto err1;
215
216	mr->umem = umem;
217	mr->ibmr.type = IB_MR_TYPE_USER;
218	mr->state = RXE_MR_STATE_VALID;
219
220	return 0;
221err1:
222	free_mr_page_info(mr);
223err2:
224	ib_umem_release(umem);
225	return err;
226}
227
228int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr)
229{
230	int err;
231
232	/* always allow remote access for FMRs */
233	rxe_mr_init(RXE_ACCESS_REMOTE, mr);
234
235	err = alloc_mr_page_info(mr, max_pages);
236	if (err)
237		goto err1;
238
239	mr->state = RXE_MR_STATE_FREE;
240	mr->ibmr.type = IB_MR_TYPE_MEM_REG;
241
242	return 0;
243
244err1:
245	return err;
246}
247
248/*
249 * I) MRs with page_size >= PAGE_SIZE,
250 * Split a large MR page (mr->page_size) into multiple PAGE_SIZE
251 * sub-pages and store them in page_info, offset is always 0.
252 *
253 * Called when mr->page_size > PAGE_SIZE. Each call to rxe_set_page()
254 * represents one mr->page_size region, which we must split into
255 * (mr->page_size >> PAGE_SHIFT) individual pages.
256 *
257 * II) MRs with page_size < PAGE_SIZE,
258 * Save each PAGE_SIZE page and its offset within the system page in page_info.
259 */
260static int rxe_set_page(struct ib_mr *ibmr, u64 dma_addr)
261{
262	struct rxe_mr *mr = to_rmr(ibmr);
263	bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT);
264	u32 i, pages_per_mr = mr_page_size(mr) >> PAGE_SHIFT;
265
266	pages_per_mr = MAX(1, pages_per_mr);
267
268	for (i = 0; i < pages_per_mr; i++) {
269		u64 addr = dma_addr + i * PAGE_SIZE;
270		struct page *sub_page = ib_virt_dma_to_page(addr);
271
272		if (unlikely(mr->nbuf >= mr->max_allowed_buf))
273			return -ENOMEM;
274
275		if (persistent && !is_pmem_page(sub_page)) {
276			rxe_dbg_mr(mr, "Page cannot be persistent\n");
277			return -EINVAL;
278		}
279
280		mr->page_info[mr->nbuf].page = sub_page;
281		mr->page_info[mr->nbuf].offset = addr & (PAGE_SIZE - 1);
282		mr->nbuf++;
283	}
284
285	return 0;
286}
287
288int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sgl,
289		  int sg_nents, unsigned int *sg_offset)
290{
291	struct rxe_mr *mr = to_rmr(ibmr);
292	unsigned int page_size = mr_page_size(mr);
293
294	/*
295	 * Ensure page_size and PAGE_SIZE are compatible for mapping.
296	 * We require one to be a multiple of the other for correct
297	 * iova-to-page conversion.
298	 */
299	if (!IS_ALIGNED(page_size, PAGE_SIZE) &&
300	    !IS_ALIGNED(PAGE_SIZE, page_size)) {
301		rxe_dbg_mr(mr, "MR page size %u must be compatible with PAGE_SIZE %lu\n",
302			   page_size, PAGE_SIZE);
303		return -EINVAL;
304	}
305
306	if (mr_page_size(mr) > PAGE_SIZE) {
307		/* resize page_info if needed */
308		u32 map_mr_pages = (page_size >> PAGE_SHIFT) * mr->num_buf;
309
310		if (map_mr_pages > mr->max_allowed_buf) {
311			rxe_dbg_mr(mr, "requested pages %u exceed max %u\n",
312				   map_mr_pages, mr->max_allowed_buf);
313			free_mr_page_info(mr);
314			if (__alloc_mr_page_info(mr, map_mr_pages))
315				return -ENOMEM;
316		}
317	}
318
319	mr->nbuf = 0;
320	mr->page_shift = ilog2(page_size);
321	mr->page_mask = ~((u64)page_size - 1);
322	mr->page_offset = mr->ibmr.iova & (page_size - 1);
323
324	return ib_sg_to_pages(ibmr, sgl, sg_nents, sg_offset, rxe_set_page);
325}
326
327static int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr,
328			      unsigned int length, enum rxe_mr_copy_dir dir)
329{
330	unsigned int bytes;
331	u8 *va;
332
333	while (length) {
334		unsigned long index = rxe_mr_iova_to_index(mr, iova);
335		struct rxe_mr_page *info = &mr->page_info[index];
336		unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova);
337
338		if (!info->page)
339			return -EFAULT;
340
341		page_offset += info->offset;
342		bytes = min_t(unsigned int, length, PAGE_SIZE - page_offset);
343		va = kmap_local_page(info->page);
344
345		if (dir == RXE_FROM_MR_OBJ)
346			memcpy(addr, va + page_offset, bytes);
347		else
348			memcpy(va + page_offset, addr, bytes);
349		kunmap_local(va);
350
351		addr += bytes;
352		iova += bytes;
353		length -= bytes;
354	}
355
356	return 0;
357}
358
359static void rxe_mr_copy_dma(struct rxe_mr *mr, u64 dma_addr, void *addr,
360			    unsigned int length, enum rxe_mr_copy_dir dir)
361{
362	unsigned int page_offset = dma_addr & (PAGE_SIZE - 1);
363	unsigned int bytes;
364	struct page *page;
365	u8 *va;
366
367	while (length) {
368		page = ib_virt_dma_to_page(dma_addr);
369		bytes = min_t(unsigned int, length,
370				PAGE_SIZE - page_offset);
371		va = kmap_local_page(page);
372
373		if (dir == RXE_TO_MR_OBJ)
374			memcpy(va + page_offset, addr, bytes);
375		else
376			memcpy(addr, va + page_offset, bytes);
377
378		kunmap_local(va);
379		page_offset = 0;
380		dma_addr += bytes;
381		addr += bytes;
382		length -= bytes;
383	}
384}
385
386int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
387		unsigned int length, enum rxe_mr_copy_dir dir)
388{
389	int err;
390
391	if (length == 0)
392		return 0;
393
394	if (WARN_ON(!mr))
395		return -EINVAL;
396
397	if (mr->ibmr.type == IB_MR_TYPE_DMA) {
398		rxe_mr_copy_dma(mr, iova, addr, length, dir);
399		return 0;
400	}
401
402	err = mr_check_range(mr, iova, length);
403	if (unlikely(err)) {
404		rxe_dbg_mr(mr, "iova out of range\n");
405		return err;
406	}
407
408	if (is_odp_mr(mr))
409		return rxe_odp_mr_copy(mr, iova, addr, length, dir);
410	else
411		return rxe_mr_copy_xarray(mr, iova, addr, length, dir);
412}
413
414/* copy data in or out of a wqe, i.e. sg list
415 * under the control of a dma descriptor
416 */
417int copy_data(
418	struct rxe_pd		*pd,
419	int			access,
420	struct rxe_dma_info	*dma,
421	void			*addr,
422	int			length,
423	enum rxe_mr_copy_dir	dir)
424{
425	int			bytes;
426	struct rxe_sge		*sge	= &dma->sge[dma->cur_sge];
427	int			offset	= dma->sge_offset;
428	int			resid	= dma->resid;
429	struct rxe_mr		*mr	= NULL;
430	u64			iova;
431	int			err;
432
433	if (length == 0)
434		return 0;
435
436	if (length > resid) {
437		err = -EINVAL;
438		goto err2;
439	}
440
441	if (sge->length && (offset < sge->length)) {
442		mr = lookup_mr(pd, access, sge->lkey, RXE_LOOKUP_LOCAL);
443		if (!mr) {
444			err = -EINVAL;
445			goto err1;
446		}
447	}
448
449	while (length > 0) {
450		bytes = length;
451
452		if (offset >= sge->length) {
453			if (mr) {
454				rxe_put(mr);
455				mr = NULL;
456			}
457			sge++;
458			dma->cur_sge++;
459			offset = 0;
460
461			if (dma->cur_sge >= dma->num_sge) {
462				err = -ENOSPC;
463				goto err2;
464			}
465
466			if (sge->length) {
467				mr = lookup_mr(pd, access, sge->lkey,
468					       RXE_LOOKUP_LOCAL);
469				if (!mr) {
470					err = -EINVAL;
471					goto err1;
472				}
473			} else {
474				continue;
475			}
476		}
477
478		if (bytes > sge->length - offset)
479			bytes = sge->length - offset;
480
481		if (bytes > 0) {
482			iova = sge->addr + offset;
483			err = rxe_mr_copy(mr, iova, addr, bytes, dir);
484			if (err)
485				goto err2;
486
487			offset	+= bytes;
488			resid	-= bytes;
489			length	-= bytes;
490			addr	+= bytes;
491		}
492	}
493
494	dma->sge_offset = offset;
495	dma->resid	= resid;
496
497	if (mr)
498		rxe_put(mr);
499
500	return 0;
501
502err2:
503	if (mr)
504		rxe_put(mr);
505err1:
506	return err;
507}
508
509static int rxe_mr_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length)
510{
511	unsigned int bytes;
512	int err;
513	u8 *va;
514
515	err = mr_check_range(mr, iova, length);
516	if (err)
517		return err;
518
519	while (length > 0) {
520		unsigned long index = rxe_mr_iova_to_index(mr, iova);
521		struct rxe_mr_page *info = &mr->page_info[index];
522		unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova);
523
524		if (!info->page)
525			return -EFAULT;
526
527		page_offset += info->offset;
528		bytes = min_t(unsigned int, length, PAGE_SIZE - page_offset);
529
530		va = kmap_local_page(info->page);
531		arch_wb_cache_pmem(va + page_offset, bytes);
532		kunmap_local(va);
533
534		length -= bytes;
535		iova += bytes;
536	}
537
538	return 0;
539}
540
541int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 start, unsigned int length)
542{
543	int err;
544
545	/* mr must be valid even if length is zero */
546	if (WARN_ON(!mr))
547		return -EINVAL;
548
549	if (length == 0)
550		return 0;
551
552	if (mr->ibmr.type == IB_MR_TYPE_DMA)
553		return -EFAULT;
554
555	if (is_odp_mr(mr))
556		err = rxe_odp_flush_pmem_iova(mr, start, length);
557	else
558		err = rxe_mr_flush_pmem_iova(mr, start, length);
559
560	return err;
561}
562
563/* Guarantee atomicity of atomic operations at the machine level. */
564DEFINE_SPINLOCK(atomic_ops_lock);
565
566enum resp_states rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
567				     u64 compare, u64 swap_add, u64 *orig_val)
568{
569	unsigned int page_offset;
570	struct page *page;
571	u64 value;
572	u64 *va;
573
574	if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
575		rxe_dbg_mr(mr, "mr not in valid state\n");
576		return RESPST_ERR_RKEY_VIOLATION;
577	}
578
579	if (mr->ibmr.type == IB_MR_TYPE_DMA) {
580		page_offset = iova & (PAGE_SIZE - 1);
581		page = ib_virt_dma_to_page(iova);
582	} else {
583		unsigned long index;
584		int err;
585		struct rxe_mr_page *info;
586
587		err = mr_check_range(mr, iova, sizeof(value));
588		if (err) {
589			rxe_dbg_mr(mr, "iova out of range\n");
590			return RESPST_ERR_RKEY_VIOLATION;
591		}
592		page_offset = rxe_mr_iova_to_page_offset(mr, iova);
593		index = rxe_mr_iova_to_index(mr, iova);
594		info = &mr->page_info[index];
595		if (!info->page)
596			return RESPST_ERR_RKEY_VIOLATION;
597
598		page_offset += info->offset;
599		page = info->page;
600	}
601
602	if (unlikely(page_offset & 0x7)) {
603		rxe_dbg_mr(mr, "iova not aligned\n");
604		return RESPST_ERR_MISALIGNED_ATOMIC;
605	}
606
607	va = kmap_local_page(page);
608
609	spin_lock_bh(&atomic_ops_lock);
610	value = *orig_val = va[page_offset >> 3];
611
612	if (opcode == IB_OPCODE_RC_COMPARE_SWAP) {
613		if (value == compare)
614			va[page_offset >> 3] = swap_add;
615	} else {
616		value += swap_add;
617		va[page_offset >> 3] = value;
618	}
619	spin_unlock_bh(&atomic_ops_lock);
620
621	kunmap_local(va);
622
623	return RESPST_NONE;
624}
625
626enum resp_states rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
627{
628	unsigned int page_offset;
629	struct page *page;
630	u64 *va;
631
632	if (mr->ibmr.type == IB_MR_TYPE_DMA) {
633		page_offset = iova & (PAGE_SIZE - 1);
634		page = ib_virt_dma_to_page(iova);
635	} else {
636		unsigned long index;
637		int err;
638		struct rxe_mr_page *info;
639
640		/* See IBA oA19-28 */
641		err = mr_check_range(mr, iova, sizeof(value));
642		if (unlikely(err)) {
643			rxe_dbg_mr(mr, "iova out of range\n");
644			return RESPST_ERR_RKEY_VIOLATION;
645		}
646		page_offset = rxe_mr_iova_to_page_offset(mr, iova);
647		index = rxe_mr_iova_to_index(mr, iova);
648		info = &mr->page_info[index];
649		if (!info->page)
650			return RESPST_ERR_RKEY_VIOLATION;
651
652		page_offset += info->offset;
653		page = info->page;
654	}
655
656	/* See IBA A19.4.2 */
657	if (unlikely(page_offset & 0x7)) {
658		rxe_dbg_mr(mr, "misaligned address\n");
659		return RESPST_ERR_MISALIGNED_ATOMIC;
660	}
661
662	va = kmap_local_page(page);
663	/* Do atomic write after all prior operations have completed */
664	smp_store_release(&va[page_offset >> 3], value);
665	kunmap_local(va);
666
667	return RESPST_NONE;
668}
669
670int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)
671{
672	struct rxe_sge		*sge	= &dma->sge[dma->cur_sge];
673	int			offset	= dma->sge_offset;
674	int			resid	= dma->resid;
675
676	while (length) {
677		unsigned int bytes;
678
679		if (offset >= sge->length) {
680			sge++;
681			dma->cur_sge++;
682			offset = 0;
683			if (dma->cur_sge >= dma->num_sge)
684				return -ENOSPC;
685		}
686
687		bytes = length;
688
689		if (bytes > sge->length - offset)
690			bytes = sge->length - offset;
691
692		offset	+= bytes;
693		resid	-= bytes;
694		length	-= bytes;
695	}
696
697	dma->sge_offset = offset;
698	dma->resid	= resid;
699
700	return 0;
701}
702
703struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
704			 enum rxe_mr_lookup_type type)
705{
706	struct rxe_mr *mr;
707	struct rxe_dev *rxe = to_rdev(pd->ibpd.device);
708	int index = key >> 8;
709
710	mr = rxe_pool_get_index(&rxe->mr_pool, index);
711	if (!mr)
712		return NULL;
713
714	if (unlikely((type == RXE_LOOKUP_LOCAL && mr->lkey != key) ||
715		     (type == RXE_LOOKUP_REMOTE && mr->rkey != key) ||
716		     mr_pd(mr) != pd || ((access & mr->access) != access) ||
717		     mr->state != RXE_MR_STATE_VALID)) {
718		rxe_put(mr);
719		mr = NULL;
720	}
721
722	return mr;
723}
724
725int rxe_invalidate_mr(struct rxe_qp *qp, u32 key)
726{
727	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
728	struct rxe_mr *mr;
729	int remote;
730	int ret;
731
732	mr = rxe_pool_get_index(&rxe->mr_pool, key >> 8);
733	if (!mr) {
734		rxe_dbg_qp(qp, "No MR for key %#x\n", key);
735		ret = -EINVAL;
736		goto err;
737	}
738
739	remote = mr->access & RXE_ACCESS_REMOTE;
740	if (remote ? (key != mr->rkey) : (key != mr->lkey)) {
741		rxe_dbg_mr(mr, "wr key (%#x) doesn't match mr key (%#x)\n",
742			key, (remote ? mr->rkey : mr->lkey));
743		ret = -EINVAL;
744		goto err_drop_ref;
745	}
746
747	if (atomic_read(&mr->num_mw) > 0) {
748		rxe_dbg_mr(mr, "Attempt to invalidate an MR while bound to MWs\n");
749		ret = -EINVAL;
750		goto err_drop_ref;
751	}
752
753	if (unlikely(mr->ibmr.type != IB_MR_TYPE_MEM_REG)) {
754		rxe_dbg_mr(mr, "Type (%d) is wrong\n", mr->ibmr.type);
755		ret = -EINVAL;
756		goto err_drop_ref;
757	}
758
759	mr->state = RXE_MR_STATE_FREE;
760	ret = 0;
761
762err_drop_ref:
763	rxe_put(mr);
764err:
765	return ret;
766}
767
768/* user can (re)register fast MR by executing a REG_MR WQE.
769 * user is expected to hold a reference on the ib mr until the
770 * WQE completes.
771 * Once a fast MR is created this is the only way to change the
772 * private keys. It is the responsibility of the user to maintain
773 * the ib mr keys in sync with rxe mr keys.
774 */
775int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
776{
777	struct rxe_mr *mr = to_rmr(wqe->wr.wr.reg.mr);
778	u32 key = wqe->wr.wr.reg.key;
779	u32 access = wqe->wr.wr.reg.access;
780
781	/* user can only register MR in free state */
782	if (unlikely(mr->state != RXE_MR_STATE_FREE)) {
783		rxe_dbg_mr(mr, "mr->lkey = 0x%x not free\n", mr->lkey);
784		return -EINVAL;
785	}
786
787	/* user can only register mr with qp in same protection domain */
788	if (unlikely(qp->ibqp.pd != mr->ibmr.pd)) {
789		rxe_dbg_mr(mr, "qp->pd and mr->pd don't match\n");
790		return -EINVAL;
791	}
792
793	/* user is only allowed to change key portion of l/rkey */
794	if (unlikely((mr->lkey & ~0xff) != (key & ~0xff))) {
795		rxe_dbg_mr(mr, "key = 0x%x has wrong index mr->lkey = 0x%x\n",
796			key, mr->lkey);
797		return -EINVAL;
798	}
799
800	mr->access = access;
801	mr->lkey = key;
802	mr->rkey = key;
803	mr->ibmr.iova = wqe->wr.wr.reg.mr->iova;
804	mr->state = RXE_MR_STATE_VALID;
805
806	return 0;
807}
808
809void rxe_mr_cleanup(struct rxe_pool_elem *elem)
810{
811	struct rxe_mr *mr = container_of(elem, typeof(*mr), elem);
812
813	rxe_put(mr_pd(mr));
814	ib_umem_release(mr->umem);
815
816	if (mr->ibmr.type != IB_MR_TYPE_DMA)
817		free_mr_page_info(mr);
818}