ionic_pgtbl.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
  3
  4#include <linux/mman.h>
  5#include <linux/dma-mapping.h>
  6
  7#include "ionic_fw.h"
  8#include "ionic_ibdev.h"
  9
 10__le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va)
 11{
 12	u64 pg_mask = BIT_ULL(buf->page_size_log2) - 1;
 13	u64 dma;
 14
 15	if (!buf->tbl_pages)
 16		return cpu_to_le64(0);
 17
 18	if (buf->tbl_pages > 1)
 19		return cpu_to_le64(buf->tbl_dma);
 20
 21	if (buf->tbl_buf)
 22		dma = le64_to_cpu(buf->tbl_buf[0]);
 23	else
 24		dma = buf->tbl_dma;
 25
 26	return cpu_to_le64(dma + (va & pg_mask));
 27}
 28
 29__be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va)
 30{
 31	if (buf->tbl_pages > 1) {
 32		u64 pg_mask = BIT_ULL(buf->page_size_log2) - 1;
 33
 34		return cpu_to_be64(va & pg_mask);
 35	}
 36
 37	return 0;
 38}
 39
 40int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma)
 41{
 42	if (unlikely(buf->tbl_pages == buf->tbl_limit))
 43		return -ENOMEM;
 44
 45	if (buf->tbl_buf)
 46		buf->tbl_buf[buf->tbl_pages] = cpu_to_le64(dma);
 47	else
 48		buf->tbl_dma = dma;
 49
 50	++buf->tbl_pages;
 51
 52	return 0;
 53}
 54
 55static int ionic_tbl_buf_alloc(struct ionic_ibdev *dev,
 56			       struct ionic_tbl_buf *buf)
 57{
 58	int rc;
 59
 60	buf->tbl_size = buf->tbl_limit * sizeof(*buf->tbl_buf);
 61	buf->tbl_buf = kmalloc(buf->tbl_size, GFP_KERNEL);
 62	if (!buf->tbl_buf)
 63		return -ENOMEM;
 64
 65	buf->tbl_dma = dma_map_single(dev->lif_cfg.hwdev, buf->tbl_buf,
 66				      buf->tbl_size, DMA_TO_DEVICE);
 67	rc = dma_mapping_error(dev->lif_cfg.hwdev, buf->tbl_dma);
 68	if (rc) {
 69		kfree(buf->tbl_buf);
 70		return rc;
 71	}
 72
 73	return 0;
 74}
 75
 76static int ionic_pgtbl_umem(struct ionic_tbl_buf *buf, struct ib_umem *umem)
 77{
 78	struct ib_block_iter biter;
 79	u64 page_dma;
 80	int rc;
 81
 82	rdma_umem_for_each_dma_block(umem, &biter, BIT_ULL(buf->page_size_log2)) {
 83		page_dma = rdma_block_iter_dma_address(&biter);
 84		rc = ionic_pgtbl_page(buf, page_dma);
 85		if (rc)
 86			return rc;
 87	}
 88
 89	return 0;
 90}
 91
 92void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf)
 93{
 94	if (buf->tbl_buf)
 95		dma_unmap_single(dev->lif_cfg.hwdev, buf->tbl_dma,
 96				 buf->tbl_size, DMA_TO_DEVICE);
 97
 98	kfree(buf->tbl_buf);
 99	memset(buf, 0, sizeof(*buf));
100}
101
102int ionic_pgtbl_init(struct ionic_ibdev *dev,
103		     struct ionic_tbl_buf *buf,
104		     struct ib_umem *umem,
105		     dma_addr_t dma,
106		     int limit,
107		     u64 page_size)
108{
109	int rc;
110
111	memset(buf, 0, sizeof(*buf));
112
113	if (umem) {
114		limit = ib_umem_num_dma_blocks(umem, page_size);
115		buf->page_size_log2 = order_base_2(page_size);
116	}
117
118	if (limit < 1)
119		return -EINVAL;
120
121	buf->tbl_limit = limit;
122
123	/* skip pgtbl if contiguous / direct translation */
124	if (limit > 1) {
125		rc = ionic_tbl_buf_alloc(dev, buf);
126		if (rc)
127			return rc;
128	}
129
130	if (umem)
131		rc = ionic_pgtbl_umem(buf, umem);
132	else
133		rc = ionic_pgtbl_page(buf, dma);
134
135	if (rc)
136		goto err_unbuf;
137
138	return 0;
139
140err_unbuf:
141	ionic_pgtbl_unbuf(dev, buf);
142	return rc;
143}