qed_ooo.c

  1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
  2/* QLogic qed NIC Driver
  3 * Copyright (c) 2015-2017  QLogic Corporation
  4 * Copyright (c) 2019-2020 Marvell International Ltd.
  5 */
  6
  7#include <linux/types.h>
  8#include <linux/dma-mapping.h>
  9#include <linux/kernel.h>
 10#include <linux/list.h>
 11#include <linux/pci.h>
 12#include <linux/slab.h>
 13#include <linux/string.h>
 14#include "qed.h"
 15#include "qed_iscsi.h"
 16#include "qed_ll2.h"
 17#include "qed_ooo.h"
 18#include "qed_cxt.h"
 19#include "qed_nvmetcp.h"
 20static struct qed_ooo_archipelago
 21*qed_ooo_seek_archipelago(struct qed_hwfn *p_hwfn,
 22			  struct qed_ooo_info
 23			  *p_ooo_info,
 24			  u32 cid)
 25{
 26	u32 idx = (cid & 0xffff) - p_ooo_info->cid_base;
 27	struct qed_ooo_archipelago *p_archipelago;
 28
 29	if (unlikely(idx >= p_ooo_info->max_num_archipelagos))
 30		return NULL;
 31
 32	p_archipelago = &p_ooo_info->p_archipelagos_mem[idx];
 33
 34	if (unlikely(list_empty(&p_archipelago->isles_list)))
 35		return NULL;
 36
 37	return p_archipelago;
 38}
 39
 40static struct qed_ooo_isle *qed_ooo_seek_isle(struct qed_hwfn *p_hwfn,
 41					      struct qed_ooo_info *p_ooo_info,
 42					      u32 cid, u8 isle)
 43{
 44	struct qed_ooo_archipelago *p_archipelago = NULL;
 45	struct qed_ooo_isle *p_isle = NULL;
 46	u8 the_num_of_isle = 1;
 47
 48	p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);
 49	if (unlikely(!p_archipelago)) {
 50		DP_NOTICE(p_hwfn,
 51			  "Connection %d is not found in OOO list\n", cid);
 52		return NULL;
 53	}
 54
 55	list_for_each_entry(p_isle, &p_archipelago->isles_list, list_entry) {
 56		if (the_num_of_isle == isle)
 57			return p_isle;
 58		the_num_of_isle++;
 59	}
 60
 61	return NULL;
 62}
 63
 64void qed_ooo_save_history_entry(struct qed_hwfn *p_hwfn,
 65				struct qed_ooo_info *p_ooo_info,
 66				struct ooo_opaque *p_cqe)
 67{
 68	struct qed_ooo_history *p_history = &p_ooo_info->ooo_history;
 69
 70	if (p_history->head_idx == p_history->num_of_cqes)
 71		p_history->head_idx = 0;
 72	p_history->p_cqes[p_history->head_idx] = *p_cqe;
 73	p_history->head_idx++;
 74}
 75
 76int qed_ooo_alloc(struct qed_hwfn *p_hwfn)
 77{
 78	u16 max_num_archipelagos = 0, cid_base;
 79	struct qed_ooo_info *p_ooo_info;
 80	enum protocol_type proto;
 81	u16 max_num_isles = 0;
 82	u32 i;
 83
 84	switch (p_hwfn->hw_info.personality) {
 85	case QED_PCI_ISCSI:
 86	case QED_PCI_NVMETCP:
 87		proto = PROTOCOLID_TCP_ULP;
 88		break;
 89	case QED_PCI_ETH_RDMA:
 90	case QED_PCI_ETH_IWARP:
 91		proto = PROTOCOLID_IWARP;
 92		break;
 93	default:
 94		DP_NOTICE(p_hwfn,
 95			  "Failed to allocate qed_ooo_info: unknown personality\n");
 96		return -EINVAL;
 97	}
 98
 99	max_num_archipelagos = (u16)qed_cxt_get_proto_cid_count(p_hwfn, proto,
100								NULL);
101	max_num_isles = QED_MAX_NUM_ISLES + max_num_archipelagos;
102	cid_base = (u16)qed_cxt_get_proto_cid_start(p_hwfn, proto);
103
104	if (!max_num_archipelagos) {
105		DP_NOTICE(p_hwfn,
106			  "Failed to allocate qed_ooo_info: unknown amount of connections\n");
107		return -EINVAL;
108	}
109
110	p_ooo_info = kzalloc(sizeof(*p_ooo_info), GFP_KERNEL);
111	if (!p_ooo_info)
112		return -ENOMEM;
113
114	p_ooo_info->cid_base = cid_base;
115	p_ooo_info->max_num_archipelagos = max_num_archipelagos;
116
117	INIT_LIST_HEAD(&p_ooo_info->free_buffers_list);
118	INIT_LIST_HEAD(&p_ooo_info->ready_buffers_list);
119	INIT_LIST_HEAD(&p_ooo_info->free_isles_list);
120
121	p_ooo_info->p_isles_mem = kcalloc(max_num_isles,
122					  sizeof(struct qed_ooo_isle),
123					  GFP_KERNEL);
124	if (!p_ooo_info->p_isles_mem)
125		goto no_isles_mem;
126
127	for (i = 0; i < max_num_isles; i++) {
128		INIT_LIST_HEAD(&p_ooo_info->p_isles_mem[i].buffers_list);
129		list_add_tail(&p_ooo_info->p_isles_mem[i].list_entry,
130			      &p_ooo_info->free_isles_list);
131	}
132
133	p_ooo_info->p_archipelagos_mem =
134				kcalloc(max_num_archipelagos,
135					sizeof(struct qed_ooo_archipelago),
136					GFP_KERNEL);
137	if (!p_ooo_info->p_archipelagos_mem)
138		goto no_archipelagos_mem;
139
140	for (i = 0; i < max_num_archipelagos; i++)
141		INIT_LIST_HEAD(&p_ooo_info->p_archipelagos_mem[i].isles_list);
142
143	p_ooo_info->ooo_history.p_cqes =
144				kcalloc(QED_MAX_NUM_OOO_HISTORY_ENTRIES,
145					sizeof(struct ooo_opaque),
146					GFP_KERNEL);
147	if (!p_ooo_info->ooo_history.p_cqes)
148		goto no_history_mem;
149
150	p_ooo_info->ooo_history.num_of_cqes = QED_MAX_NUM_OOO_HISTORY_ENTRIES;
151
152	p_hwfn->p_ooo_info = p_ooo_info;
153	return 0;
154
155no_history_mem:
156	kfree(p_ooo_info->p_archipelagos_mem);
157no_archipelagos_mem:
158	kfree(p_ooo_info->p_isles_mem);
159no_isles_mem:
160	kfree(p_ooo_info);
161	return -ENOMEM;
162}
163
164void qed_ooo_release_connection_isles(struct qed_hwfn *p_hwfn,
165				      struct qed_ooo_info *p_ooo_info, u32 cid)
166{
167	struct qed_ooo_archipelago *p_archipelago;
168	struct qed_ooo_buffer *p_buffer;
169	struct qed_ooo_isle *p_isle;
170
171	p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);
172	if (!p_archipelago)
173		return;
174
175	while (!list_empty(&p_archipelago->isles_list)) {
176		p_isle = list_first_entry(&p_archipelago->isles_list,
177					  struct qed_ooo_isle, list_entry);
178
179		list_del(&p_isle->list_entry);
180
181		while (!list_empty(&p_isle->buffers_list)) {
182			p_buffer = list_first_entry(&p_isle->buffers_list,
183						    struct qed_ooo_buffer,
184						    list_entry);
185
186			list_move_tail(&p_buffer->list_entry,
187				       &p_ooo_info->free_buffers_list);
188		}
189		list_add_tail(&p_isle->list_entry,
190			      &p_ooo_info->free_isles_list);
191	}
192}
193
194void qed_ooo_release_all_isles(struct qed_hwfn *p_hwfn,
195			       struct qed_ooo_info *p_ooo_info)
196{
197	struct qed_ooo_archipelago *p_archipelago;
198	struct qed_ooo_buffer *p_buffer;
199	struct qed_ooo_isle *p_isle;
200	u32 i;
201
202	for (i = 0; i < p_ooo_info->max_num_archipelagos; i++) {
203		p_archipelago = &(p_ooo_info->p_archipelagos_mem[i]);
204
205		while (!list_empty(&p_archipelago->isles_list)) {
206			p_isle = list_first_entry(&p_archipelago->isles_list,
207						  struct qed_ooo_isle,
208						  list_entry);
209
210			list_del(&p_isle->list_entry);
211
212			while (!list_empty(&p_isle->buffers_list)) {
213				p_buffer =
214				    list_first_entry(&p_isle->buffers_list,
215						     struct qed_ooo_buffer,
216						     list_entry);
217
218				list_move_tail(&p_buffer->list_entry,
219					       &p_ooo_info->free_buffers_list);
220			}
221			list_add_tail(&p_isle->list_entry,
222				      &p_ooo_info->free_isles_list);
223		}
224	}
225	if (!list_empty(&p_ooo_info->ready_buffers_list))
226		list_splice_tail_init(&p_ooo_info->ready_buffers_list,
227				      &p_ooo_info->free_buffers_list);
228}
229
230void qed_ooo_setup(struct qed_hwfn *p_hwfn)
231{
232	qed_ooo_release_all_isles(p_hwfn, p_hwfn->p_ooo_info);
233	memset(p_hwfn->p_ooo_info->ooo_history.p_cqes, 0,
234	       p_hwfn->p_ooo_info->ooo_history.num_of_cqes *
235	       sizeof(struct ooo_opaque));
236	p_hwfn->p_ooo_info->ooo_history.head_idx = 0;
237}
238
239void qed_ooo_free(struct qed_hwfn *p_hwfn)
240{
241	struct qed_ooo_info *p_ooo_info  = p_hwfn->p_ooo_info;
242	struct qed_ooo_buffer *p_buffer;
243
244	if (!p_ooo_info)
245		return;
246
247	qed_ooo_release_all_isles(p_hwfn, p_ooo_info);
248	while (!list_empty(&p_ooo_info->free_buffers_list)) {
249		p_buffer = list_first_entry(&p_ooo_info->free_buffers_list,
250					    struct qed_ooo_buffer, list_entry);
251
252		list_del(&p_buffer->list_entry);
253		dma_free_coherent(&p_hwfn->cdev->pdev->dev,
254				  p_buffer->rx_buffer_size,
255				  p_buffer->rx_buffer_virt_addr,
256				  p_buffer->rx_buffer_phys_addr);
257		kfree(p_buffer);
258	}
259
260	kfree(p_ooo_info->p_isles_mem);
261	kfree(p_ooo_info->p_archipelagos_mem);
262	kfree(p_ooo_info->ooo_history.p_cqes);
263	kfree(p_ooo_info);
264	p_hwfn->p_ooo_info = NULL;
265}
266
267void qed_ooo_put_free_buffer(struct qed_hwfn *p_hwfn,
268			     struct qed_ooo_info *p_ooo_info,
269			     struct qed_ooo_buffer *p_buffer)
270{
271	list_add_tail(&p_buffer->list_entry, &p_ooo_info->free_buffers_list);
272}
273
274struct qed_ooo_buffer *qed_ooo_get_free_buffer(struct qed_hwfn *p_hwfn,
275					       struct qed_ooo_info *p_ooo_info)
276{
277	struct qed_ooo_buffer *p_buffer = NULL;
278
279	if (!list_empty(&p_ooo_info->free_buffers_list)) {
280		p_buffer = list_first_entry(&p_ooo_info->free_buffers_list,
281					    struct qed_ooo_buffer, list_entry);
282
283		list_del(&p_buffer->list_entry);
284	}
285
286	return p_buffer;
287}
288
289void qed_ooo_put_ready_buffer(struct qed_hwfn *p_hwfn,
290			      struct qed_ooo_info *p_ooo_info,
291			      struct qed_ooo_buffer *p_buffer, u8 on_tail)
292{
293	if (on_tail)
294		list_add_tail(&p_buffer->list_entry,
295			      &p_ooo_info->ready_buffers_list);
296	else
297		list_add(&p_buffer->list_entry,
298			 &p_ooo_info->ready_buffers_list);
299}
300
301struct qed_ooo_buffer *qed_ooo_get_ready_buffer(struct qed_hwfn *p_hwfn,
302						struct qed_ooo_info *p_ooo_info)
303{
304	struct qed_ooo_buffer *p_buffer = NULL;
305
306	if (!list_empty(&p_ooo_info->ready_buffers_list)) {
307		p_buffer = list_first_entry(&p_ooo_info->ready_buffers_list,
308					    struct qed_ooo_buffer, list_entry);
309
310		list_del(&p_buffer->list_entry);
311	}
312
313	return p_buffer;
314}
315
316void qed_ooo_delete_isles(struct qed_hwfn *p_hwfn,
317			  struct qed_ooo_info *p_ooo_info,
318			  u32 cid, u8 drop_isle, u8 drop_size)
319{
320	struct qed_ooo_isle *p_isle = NULL;
321	u8 isle_idx;
322
323	for (isle_idx = 0; isle_idx < drop_size; isle_idx++) {
324		p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, drop_isle);
325		if (!p_isle) {
326			DP_NOTICE(p_hwfn,
327				  "Isle %d is not found(cid %d)\n",
328				  drop_isle, cid);
329			return;
330		}
331		if (list_empty(&p_isle->buffers_list))
332			DP_NOTICE(p_hwfn,
333				  "Isle %d is empty(cid %d)\n", drop_isle, cid);
334		else
335			list_splice_tail_init(&p_isle->buffers_list,
336					      &p_ooo_info->free_buffers_list);
337
338		list_del(&p_isle->list_entry);
339		p_ooo_info->cur_isles_number--;
340		list_add(&p_isle->list_entry, &p_ooo_info->free_isles_list);
341	}
342}
343
344void qed_ooo_add_new_isle(struct qed_hwfn *p_hwfn,
345			  struct qed_ooo_info *p_ooo_info,
346			  u32 cid, u8 ooo_isle,
347			  struct qed_ooo_buffer *p_buffer)
348{
349	struct qed_ooo_archipelago *p_archipelago = NULL;
350	struct qed_ooo_isle *p_prev_isle = NULL;
351	struct qed_ooo_isle *p_isle = NULL;
352
353	if (ooo_isle > 1) {
354		p_prev_isle = qed_ooo_seek_isle(p_hwfn,
355						p_ooo_info, cid, ooo_isle - 1);
356		if (unlikely(!p_prev_isle)) {
357			DP_NOTICE(p_hwfn,
358				  "Isle %d is not found(cid %d)\n",
359				  ooo_isle - 1, cid);
360			return;
361		}
362	}
363	p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);
364	if (unlikely(!p_archipelago && ooo_isle != 1)) {
365		DP_NOTICE(p_hwfn,
366			  "Connection %d is not found in OOO list\n", cid);
367		return;
368	}
369
370	if (!list_empty(&p_ooo_info->free_isles_list)) {
371		p_isle = list_first_entry(&p_ooo_info->free_isles_list,
372					  struct qed_ooo_isle, list_entry);
373
374		list_del(&p_isle->list_entry);
375		if (unlikely(!list_empty(&p_isle->buffers_list))) {
376			DP_NOTICE(p_hwfn, "Free isle is not empty\n");
377			INIT_LIST_HEAD(&p_isle->buffers_list);
378		}
379	} else {
380		DP_NOTICE(p_hwfn, "No more free isles\n");
381		return;
382	}
383
384	if (!p_archipelago) {
385		u32 idx = (cid & 0xffff) - p_ooo_info->cid_base;
386
387		p_archipelago = &p_ooo_info->p_archipelagos_mem[idx];
388	}
389
390	list_add(&p_buffer->list_entry, &p_isle->buffers_list);
391	p_ooo_info->cur_isles_number++;
392	p_ooo_info->gen_isles_number++;
393
394	if (p_ooo_info->cur_isles_number > p_ooo_info->max_isles_number)
395		p_ooo_info->max_isles_number = p_ooo_info->cur_isles_number;
396
397	if (!p_prev_isle)
398		list_add(&p_isle->list_entry, &p_archipelago->isles_list);
399	else
400		list_add(&p_isle->list_entry, &p_prev_isle->list_entry);
401}
402
403void qed_ooo_add_new_buffer(struct qed_hwfn *p_hwfn,
404			    struct qed_ooo_info *p_ooo_info,
405			    u32 cid,
406			    u8 ooo_isle,
407			    struct qed_ooo_buffer *p_buffer, u8 buffer_side)
408{
409	struct qed_ooo_isle *p_isle = NULL;
410
411	p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, ooo_isle);
412	if (unlikely(!p_isle)) {
413		DP_NOTICE(p_hwfn,
414			  "Isle %d is not found(cid %d)\n", ooo_isle, cid);
415		return;
416	}
417
418	if (unlikely(buffer_side == QED_OOO_LEFT_BUF))
419		list_add(&p_buffer->list_entry, &p_isle->buffers_list);
420	else
421		list_add_tail(&p_buffer->list_entry, &p_isle->buffers_list);
422}
423
424void qed_ooo_join_isles(struct qed_hwfn *p_hwfn,
425			struct qed_ooo_info *p_ooo_info, u32 cid, u8 left_isle)
426{
427	struct qed_ooo_isle *p_right_isle = NULL;
428	struct qed_ooo_isle *p_left_isle = NULL;
429
430	p_right_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid,
431					 left_isle + 1);
432	if (unlikely(!p_right_isle)) {
433		DP_NOTICE(p_hwfn,
434			  "Right isle %d is not found(cid %d)\n",
435			  left_isle + 1, cid);
436		return;
437	}
438
439	list_del(&p_right_isle->list_entry);
440	p_ooo_info->cur_isles_number--;
441	if (left_isle) {
442		p_left_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid,
443						left_isle);
444		if (unlikely(!p_left_isle)) {
445			DP_NOTICE(p_hwfn,
446				  "Left isle %d is not found(cid %d)\n",
447				  left_isle, cid);
448			return;
449		}
450		list_splice_tail_init(&p_right_isle->buffers_list,
451				      &p_left_isle->buffers_list);
452	} else {
453		list_splice_tail_init(&p_right_isle->buffers_list,
454				      &p_ooo_info->ready_buffers_list);
455	}
456	list_add_tail(&p_right_isle->list_entry, &p_ooo_info->free_isles_list);
457}