gve_buffer_mgmt_dqo.c

  1// SPDX-License-Identifier: (GPL-2.0 OR MIT)
  2/* Google virtual Ethernet (gve) driver
  3 *
  4 * Copyright (C) 2015-2024 Google, Inc.
  5 */
  6
  7#include <net/xdp_sock_drv.h>
  8#include "gve.h"
  9#include "gve_utils.h"
 10
 11int gve_buf_ref_cnt(struct gve_rx_buf_state_dqo *bs)
 12{
 13	return page_count(bs->page_info.page) - bs->page_info.pagecnt_bias;
 14}
 15
 16struct gve_rx_buf_state_dqo *gve_alloc_buf_state(struct gve_rx_ring *rx)
 17{
 18	struct gve_rx_buf_state_dqo *buf_state;
 19	s16 buffer_id;
 20
 21	buffer_id = rx->dqo.free_buf_states;
 22	if (unlikely(buffer_id == -1))
 23		return NULL;
 24
 25	buf_state = &rx->dqo.buf_states[buffer_id];
 26
 27	/* Remove buf_state from free list */
 28	rx->dqo.free_buf_states = buf_state->next;
 29
 30	/* Point buf_state to itself to mark it as allocated */
 31	buf_state->next = buffer_id;
 32
 33	/* Clear the buffer pointers */
 34	buf_state->page_info.page = NULL;
 35	buf_state->xsk_buff = NULL;
 36
 37	return buf_state;
 38}
 39
 40bool gve_buf_state_is_allocated(struct gve_rx_ring *rx,
 41				struct gve_rx_buf_state_dqo *buf_state)
 42{
 43	s16 buffer_id = buf_state - rx->dqo.buf_states;
 44
 45	return buf_state->next == buffer_id;
 46}
 47
 48void gve_free_buf_state(struct gve_rx_ring *rx,
 49			struct gve_rx_buf_state_dqo *buf_state)
 50{
 51	s16 buffer_id = buf_state - rx->dqo.buf_states;
 52
 53	buf_state->next = rx->dqo.free_buf_states;
 54	rx->dqo.free_buf_states = buffer_id;
 55}
 56
 57struct gve_rx_buf_state_dqo *gve_dequeue_buf_state(struct gve_rx_ring *rx,
 58						   struct gve_index_list *list)
 59{
 60	struct gve_rx_buf_state_dqo *buf_state;
 61	s16 buffer_id;
 62
 63	buffer_id = list->head;
 64	if (unlikely(buffer_id == -1))
 65		return NULL;
 66
 67	buf_state = &rx->dqo.buf_states[buffer_id];
 68
 69	/* Remove buf_state from list */
 70	list->head = buf_state->next;
 71	if (buf_state->next == -1)
 72		list->tail = -1;
 73
 74	/* Point buf_state to itself to mark it as allocated */
 75	buf_state->next = buffer_id;
 76
 77	return buf_state;
 78}
 79
 80void gve_enqueue_buf_state(struct gve_rx_ring *rx, struct gve_index_list *list,
 81			   struct gve_rx_buf_state_dqo *buf_state)
 82{
 83	s16 buffer_id = buf_state - rx->dqo.buf_states;
 84
 85	buf_state->next = -1;
 86
 87	if (list->head == -1) {
 88		list->head = buffer_id;
 89		list->tail = buffer_id;
 90	} else {
 91		int tail = list->tail;
 92
 93		rx->dqo.buf_states[tail].next = buffer_id;
 94		list->tail = buffer_id;
 95	}
 96}
 97
 98struct gve_rx_buf_state_dqo *gve_get_recycled_buf_state(struct gve_rx_ring *rx)
 99{
100	struct gve_rx_buf_state_dqo *buf_state;
101	int i;
102
103	/* Recycled buf states are immediately usable. */
104	buf_state = gve_dequeue_buf_state(rx, &rx->dqo.recycled_buf_states);
105	if (likely(buf_state))
106		return buf_state;
107
108	if (unlikely(rx->dqo.used_buf_states.head == -1))
109		return NULL;
110
111	/* Used buf states are only usable when ref count reaches 0, which means
112	 * no SKBs refer to them.
113	 *
114	 * Search a limited number before giving up.
115	 */
116	for (i = 0; i < 5; i++) {
117		buf_state = gve_dequeue_buf_state(rx, &rx->dqo.used_buf_states);
118		if (gve_buf_ref_cnt(buf_state) == 0) {
119			rx->dqo.used_buf_states_cnt--;
120			return buf_state;
121		}
122
123		gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state);
124	}
125
126	return NULL;
127}
128
129int gve_alloc_qpl_page_dqo(struct gve_rx_ring *rx,
130			   struct gve_rx_buf_state_dqo *buf_state)
131{
132	struct gve_priv *priv = rx->gve;
133	u32 idx;
134
135	idx = rx->dqo.next_qpl_page_idx;
136	if (idx >= gve_get_rx_pages_per_qpl_dqo(priv->rx_desc_cnt)) {
137		net_err_ratelimited("%s: Out of QPL pages\n",
138				    priv->dev->name);
139		return -ENOMEM;
140	}
141	buf_state->page_info.page = rx->dqo.qpl->pages[idx];
142	buf_state->addr = rx->dqo.qpl->page_buses[idx];
143	rx->dqo.next_qpl_page_idx++;
144	buf_state->page_info.page_offset = 0;
145	buf_state->page_info.page_address =
146		page_address(buf_state->page_info.page);
147	buf_state->page_info.buf_size = rx->packet_buffer_truesize;
148	buf_state->page_info.pad = rx->rx_headroom;
149	buf_state->last_single_ref_offset = 0;
150
151	/* The page already has 1 ref. */
152	page_ref_add(buf_state->page_info.page, INT_MAX - 1);
153	buf_state->page_info.pagecnt_bias = INT_MAX;
154
155	return 0;
156}
157
158void gve_free_qpl_page_dqo(struct gve_rx_buf_state_dqo *buf_state)
159{
160	if (!buf_state->page_info.page)
161		return;
162
163	page_ref_sub(buf_state->page_info.page,
164		     buf_state->page_info.pagecnt_bias - 1);
165	buf_state->page_info.page = NULL;
166}
167
168void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx,
169			 struct gve_rx_buf_state_dqo *buf_state)
170{
171	const u16 data_buffer_size = rx->packet_buffer_truesize;
172	int pagecount;
173
174	/* Can't reuse if we only fit one buffer per page */
175	if (data_buffer_size * 2 > PAGE_SIZE)
176		goto mark_used;
177
178	pagecount = gve_buf_ref_cnt(buf_state);
179
180	/* Record the offset when we have a single remaining reference.
181	 *
182	 * When this happens, we know all of the other offsets of the page are
183	 * usable.
184	 */
185	if (pagecount == 1) {
186		buf_state->last_single_ref_offset =
187			buf_state->page_info.page_offset;
188	}
189
190	/* Use the next buffer sized chunk in the page. */
191	buf_state->page_info.page_offset += data_buffer_size;
192	buf_state->page_info.page_offset &= (PAGE_SIZE - 1);
193
194	/* If we wrap around to the same offset without ever dropping to 1
195	 * reference, then we don't know if this offset was ever freed.
196	 */
197	if (buf_state->page_info.page_offset ==
198	    buf_state->last_single_ref_offset) {
199		goto mark_used;
200	}
201
202	gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, buf_state);
203	return;
204
205mark_used:
206	gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state);
207	rx->dqo.used_buf_states_cnt++;
208}
209
210void gve_free_to_page_pool(struct gve_rx_ring *rx,
211			   struct gve_rx_buf_state_dqo *buf_state,
212			   bool allow_direct)
213{
214	netmem_ref netmem = buf_state->page_info.netmem;
215
216	if (!netmem)
217		return;
218
219	page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, allow_direct);
220	buf_state->page_info.netmem = 0;
221}
222
223static int gve_alloc_from_page_pool(struct gve_rx_ring *rx,
224				    struct gve_rx_buf_state_dqo *buf_state)
225{
226	netmem_ref netmem;
227
228	buf_state->page_info.buf_size = rx->packet_buffer_truesize;
229	netmem = page_pool_alloc_netmem(rx->dqo.page_pool,
230					&buf_state->page_info.page_offset,
231					&buf_state->page_info.buf_size,
232					GFP_ATOMIC);
233
234	if (!netmem)
235		return -ENOMEM;
236
237	buf_state->page_info.netmem = netmem;
238	buf_state->page_info.page_address = netmem_address(netmem);
239	buf_state->addr = page_pool_get_dma_addr_netmem(netmem);
240	buf_state->page_info.pad = rx->dqo.page_pool->p.offset;
241
242	return 0;
243}
244
245struct page_pool *gve_rx_create_page_pool(struct gve_priv *priv,
246					  struct gve_rx_ring *rx,
247					  bool xdp)
248{
249	u32 ntfy_id = gve_rx_idx_to_ntfy(priv, rx->q_num);
250	struct page_pool_params pp = {
251		.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
252		.order = 0,
253		.pool_size = GVE_PAGE_POOL_SIZE_MULTIPLIER * priv->rx_desc_cnt,
254		.nid = priv->numa_node,
255		.dev = &priv->pdev->dev,
256		.netdev = priv->dev,
257		.napi = &priv->ntfy_blocks[ntfy_id].napi,
258		.max_len = PAGE_SIZE,
259		.dma_dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE,
260		.offset = xdp ? XDP_PACKET_HEADROOM : 0,
261	};
262
263	if (priv->header_split_enabled) {
264		pp.flags |= PP_FLAG_ALLOW_UNREADABLE_NETMEM;
265		pp.queue_idx = rx->q_num;
266	}
267
268	return page_pool_create(&pp);
269}
270
271void gve_free_buffer(struct gve_rx_ring *rx,
272		     struct gve_rx_buf_state_dqo *buf_state)
273{
274	if (rx->dqo.page_pool) {
275		gve_free_to_page_pool(rx, buf_state, true);
276		gve_free_buf_state(rx, buf_state);
277	} else {
278		gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states,
279				      buf_state);
280	}
281}
282
283void gve_reuse_buffer(struct gve_rx_ring *rx,
284		      struct gve_rx_buf_state_dqo *buf_state)
285{
286	if (rx->dqo.page_pool) {
287		buf_state->page_info.netmem = 0;
288		gve_free_buf_state(rx, buf_state);
289	} else {
290		gve_dec_pagecnt_bias(&buf_state->page_info);
291		gve_try_recycle_buf(rx->gve, rx, buf_state);
292	}
293}
294
295int gve_alloc_buffer(struct gve_rx_ring *rx, struct gve_rx_desc_dqo *desc)
296{
297	struct gve_rx_buf_state_dqo *buf_state;
298
299	if (rx->xsk_pool) {
300		buf_state = gve_alloc_buf_state(rx);
301		if (unlikely(!buf_state))
302			return -ENOMEM;
303
304		buf_state->xsk_buff = xsk_buff_alloc(rx->xsk_pool);
305		if (unlikely(!buf_state->xsk_buff)) {
306			xsk_set_rx_need_wakeup(rx->xsk_pool);
307			gve_free_buf_state(rx, buf_state);
308			return -ENOMEM;
309		}
310		/* Allocated xsk buffer. Clear wakeup in case it was set. */
311		xsk_clear_rx_need_wakeup(rx->xsk_pool);
312		desc->buf_id = cpu_to_le16(buf_state - rx->dqo.buf_states);
313		desc->buf_addr =
314			cpu_to_le64(xsk_buff_xdp_get_dma(buf_state->xsk_buff));
315		return 0;
316	} else if (rx->dqo.page_pool) {
317		buf_state = gve_alloc_buf_state(rx);
318		if (WARN_ON_ONCE(!buf_state))
319			return -ENOMEM;
320
321		if (gve_alloc_from_page_pool(rx, buf_state))
322			goto free_buf_state;
323	} else {
324		buf_state = gve_get_recycled_buf_state(rx);
325		if (unlikely(!buf_state)) {
326			buf_state = gve_alloc_buf_state(rx);
327			if (unlikely(!buf_state))
328				return -ENOMEM;
329
330			if (unlikely(gve_alloc_qpl_page_dqo(rx, buf_state)))
331				goto free_buf_state;
332		}
333	}
334	desc->buf_id = cpu_to_le16(buf_state - rx->dqo.buf_states);
335	desc->buf_addr = cpu_to_le64(buf_state->addr +
336				     buf_state->page_info.page_offset +
337				     buf_state->page_info.pad);
338
339	return 0;
340
341free_buf_state:
342	gve_free_buf_state(rx, buf_state);
343	return -ENOMEM;
344}