gdma.h

  1/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
  2/* Copyright (c) 2021, Microsoft Corporation. */
  3
  4#ifndef _GDMA_H
  5#define _GDMA_H
  6
  7#include <linux/dma-mapping.h>
  8#include <linux/netdevice.h>
  9
 10#include "shm_channel.h"
 11
 12#define GDMA_STATUS_MORE_ENTRIES	0x00000105
 13#define GDMA_STATUS_CMD_UNSUPPORTED	0xffffffff
 14
 15/* Structures labeled with "HW DATA" are exchanged with the hardware. All of
 16 * them are naturally aligned and hence don't need __packed.
 17 */
 18
 19enum gdma_request_type {
 20	GDMA_VERIFY_VF_DRIVER_VERSION	= 1,
 21	GDMA_QUERY_MAX_RESOURCES	= 2,
 22	GDMA_LIST_DEVICES		= 3,
 23	GDMA_REGISTER_DEVICE		= 4,
 24	GDMA_DEREGISTER_DEVICE		= 5,
 25	GDMA_GENERATE_TEST_EQE		= 10,
 26	GDMA_CREATE_QUEUE		= 12,
 27	GDMA_DISABLE_QUEUE		= 13,
 28	GDMA_ALLOCATE_RESOURCE_RANGE	= 22,
 29	GDMA_DESTROY_RESOURCE_RANGE	= 24,
 30	GDMA_CREATE_DMA_REGION		= 25,
 31	GDMA_DMA_REGION_ADD_PAGES	= 26,
 32	GDMA_DESTROY_DMA_REGION		= 27,
 33	GDMA_CREATE_PD			= 29,
 34	GDMA_DESTROY_PD			= 30,
 35	GDMA_CREATE_MR			= 31,
 36	GDMA_DESTROY_MR			= 32,
 37	GDMA_QUERY_HWC_TIMEOUT		= 84, /* 0x54 */
 38};
 39
 40#define GDMA_RESOURCE_DOORBELL_PAGE	27
 41
 42enum gdma_queue_type {
 43	GDMA_INVALID_QUEUE,
 44	GDMA_SQ,
 45	GDMA_RQ,
 46	GDMA_CQ,
 47	GDMA_EQ,
 48};
 49
 50enum gdma_work_request_flags {
 51	GDMA_WR_NONE			= 0,
 52	GDMA_WR_OOB_IN_SGL		= BIT(0),
 53	GDMA_WR_PAD_BY_SGE0		= BIT(1),
 54};
 55
 56enum gdma_eqe_type {
 57	GDMA_EQE_COMPLETION		= 3,
 58	GDMA_EQE_TEST_EVENT		= 64,
 59	GDMA_EQE_HWC_INIT_EQ_ID_DB	= 129,
 60	GDMA_EQE_HWC_INIT_DATA		= 130,
 61	GDMA_EQE_HWC_INIT_DONE		= 131,
 62	GDMA_EQE_HWC_FPGA_RECONFIG	= 132,
 63	GDMA_EQE_HWC_SOC_RECONFIG_DATA	= 133,
 64	GDMA_EQE_HWC_SOC_SERVICE	= 134,
 65	GDMA_EQE_HWC_RESET_REQUEST	= 135,
 66	GDMA_EQE_RNIC_QP_FATAL		= 176,
 67};
 68
 69enum {
 70	GDMA_DEVICE_NONE	= 0,
 71	GDMA_DEVICE_HWC		= 1,
 72	GDMA_DEVICE_MANA	= 2,
 73	GDMA_DEVICE_MANA_IB	= 3,
 74};
 75
 76enum gdma_service_type {
 77	GDMA_SERVICE_TYPE_NONE		= 0,
 78	GDMA_SERVICE_TYPE_RDMA_SUSPEND	= 1,
 79	GDMA_SERVICE_TYPE_RDMA_RESUME	= 2,
 80};
 81
 82struct mana_service_work {
 83	struct work_struct work;
 84	struct gdma_dev *gdma_dev;
 85	enum gdma_service_type event;
 86};
 87
 88struct gdma_resource {
 89	/* Protect the bitmap */
 90	spinlock_t lock;
 91
 92	/* The bitmap size in bits. */
 93	u32 size;
 94
 95	/* The bitmap tracks the resources. */
 96	unsigned long *map;
 97};
 98
 99union gdma_doorbell_entry {
100	u64	as_uint64;
101
102	struct {
103		u64 id		: 24;
104		u64 reserved	: 8;
105		u64 tail_ptr	: 31;
106		u64 arm		: 1;
107	} cq;
108
109	struct {
110		u64 id		: 24;
111		u64 wqe_cnt	: 8;
112		u64 tail_ptr	: 32;
113	} rq;
114
115	struct {
116		u64 id		: 24;
117		u64 reserved	: 8;
118		u64 tail_ptr	: 32;
119	} sq;
120
121	struct {
122		u64 id		: 16;
123		u64 reserved	: 16;
124		u64 tail_ptr	: 31;
125		u64 arm		: 1;
126	} eq;
127}; /* HW DATA */
128
129struct gdma_msg_hdr {
130	u32 hdr_type;
131	u32 msg_type;
132	u16 msg_version;
133	u16 hwc_msg_id;
134	u32 msg_size;
135}; /* HW DATA */
136
137struct gdma_dev_id {
138	union {
139		struct {
140			u16 type;
141			u16 instance;
142		};
143
144		u32 as_uint32;
145	};
146}; /* HW DATA */
147
148struct gdma_req_hdr {
149	struct gdma_msg_hdr req;
150	struct gdma_msg_hdr resp; /* The expected response */
151	struct gdma_dev_id dev_id;
152	u32 activity_id;
153}; /* HW DATA */
154
155struct gdma_resp_hdr {
156	struct gdma_msg_hdr response;
157	struct gdma_dev_id dev_id;
158	u32 activity_id;
159	u32 status;
160	u32 reserved;
161}; /* HW DATA */
162
163struct gdma_general_req {
164	struct gdma_req_hdr hdr;
165}; /* HW DATA */
166
167#define GDMA_MESSAGE_V1 1
168#define GDMA_MESSAGE_V2 2
169#define GDMA_MESSAGE_V3 3
170#define GDMA_MESSAGE_V4 4
171
172struct gdma_general_resp {
173	struct gdma_resp_hdr hdr;
174}; /* HW DATA */
175
176#define GDMA_STANDARD_HEADER_TYPE 0
177
178static inline void mana_gd_init_req_hdr(struct gdma_req_hdr *hdr, u32 code,
179					u32 req_size, u32 resp_size)
180{
181	hdr->req.hdr_type = GDMA_STANDARD_HEADER_TYPE;
182	hdr->req.msg_type = code;
183	hdr->req.msg_version = GDMA_MESSAGE_V1;
184	hdr->req.msg_size = req_size;
185
186	hdr->resp.hdr_type = GDMA_STANDARD_HEADER_TYPE;
187	hdr->resp.msg_type = code;
188	hdr->resp.msg_version = GDMA_MESSAGE_V1;
189	hdr->resp.msg_size = resp_size;
190}
191
192/* The 16-byte struct is part of the GDMA work queue entry (WQE). */
193struct gdma_sge {
194	u64 address;
195	u32 mem_key;
196	u32 size;
197}; /* HW DATA */
198
199struct gdma_wqe_request {
200	struct gdma_sge *sgl;
201	u32 num_sge;
202
203	u32 inline_oob_size;
204	const void *inline_oob_data;
205
206	u32 flags;
207	u32 client_data_unit;
208};
209
210enum gdma_page_type {
211	GDMA_PAGE_TYPE_4K,
212};
213
214#define GDMA_INVALID_DMA_REGION 0
215
216struct gdma_mem_info {
217	struct device *dev;
218
219	dma_addr_t dma_handle;
220	void *virt_addr;
221	u64 length;
222
223	/* Allocated by the PF driver */
224	u64 dma_region_handle;
225};
226
227#define REGISTER_ATB_MST_MKEY_LOWER_SIZE 8
228
229struct gdma_dev {
230	struct gdma_context *gdma_context;
231
232	struct gdma_dev_id dev_id;
233
234	u32 pdid;
235	u32 doorbell;
236	u32 gpa_mkey;
237
238	/* GDMA driver specific pointer */
239	void *driver_data;
240
241	struct auxiliary_device *adev;
242	bool is_suspended;
243	bool rdma_teardown;
244};
245
246/* MANA_PAGE_SIZE is the DMA unit */
247#define MANA_PAGE_SHIFT 12
248#define MANA_PAGE_SIZE BIT(MANA_PAGE_SHIFT)
249#define MANA_PAGE_ALIGN(x) ALIGN((x), MANA_PAGE_SIZE)
250#define MANA_PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), MANA_PAGE_SIZE)
251#define MANA_PFN(a) ((a) >> MANA_PAGE_SHIFT)
252
253/* Required by HW */
254#define MANA_MIN_QSIZE MANA_PAGE_SIZE
255
256#define GDMA_CQE_SIZE 64
257#define GDMA_EQE_SIZE 16
258#define GDMA_MAX_SQE_SIZE 512
259#define GDMA_MAX_RQE_SIZE 256
260
261#define GDMA_COMP_DATA_SIZE 0x3C
262
263#define GDMA_EVENT_DATA_SIZE 0xC
264
265/* The WQE size must be a multiple of the Basic Unit, which is 32 bytes. */
266#define GDMA_WQE_BU_SIZE 32
267
268#define INVALID_PDID		UINT_MAX
269#define INVALID_DOORBELL	UINT_MAX
270#define INVALID_MEM_KEY		UINT_MAX
271#define INVALID_QUEUE_ID	UINT_MAX
272#define INVALID_PCI_MSIX_INDEX  UINT_MAX
273
274struct gdma_comp {
275	u32 cqe_data[GDMA_COMP_DATA_SIZE / 4];
276	u32 wq_num;
277	bool is_sq;
278};
279
280struct gdma_event {
281	u32 details[GDMA_EVENT_DATA_SIZE / 4];
282	u8  type;
283};
284
285struct gdma_queue;
286
287struct mana_eq {
288	struct gdma_queue	*eq;
289	struct dentry		*mana_eq_debugfs;
290};
291
292typedef void gdma_eq_callback(void *context, struct gdma_queue *q,
293			      struct gdma_event *e);
294
295typedef void gdma_cq_callback(void *context, struct gdma_queue *q);
296
297/* The 'head' is the producer index. For SQ/RQ, when the driver posts a WQE
298 * (Note: the WQE size must be a multiple of the 32-byte Basic Unit), the
299 * driver increases the 'head' in BUs rather than in bytes, and notifies
300 * the HW of the updated head. For EQ/CQ, the driver uses the 'head' to track
301 * the HW head, and increases the 'head' by 1 for every processed EQE/CQE.
302 *
303 * The 'tail' is the consumer index for SQ/RQ. After the CQE of the SQ/RQ is
304 * processed, the driver increases the 'tail' to indicate that WQEs have
305 * been consumed by the HW, so the driver can post new WQEs into the SQ/RQ.
306 *
307 * The driver doesn't use the 'tail' for EQ/CQ, because the driver ensures
308 * that the EQ/CQ is big enough so they can't overflow, and the driver uses
309 * the owner bits mechanism to detect if the queue has become empty.
310 */
311struct gdma_queue {
312	struct gdma_dev *gdma_dev;
313
314	enum gdma_queue_type type;
315	u32 id;
316
317	struct gdma_mem_info mem_info;
318
319	void *queue_mem_ptr;
320	u32 queue_size;
321
322	bool monitor_avl_buf;
323
324	u32 head;
325	u32 tail;
326	struct list_head entry;
327
328	/* Extra fields specific to EQ/CQ. */
329	union {
330		struct {
331			bool disable_needed;
332
333			gdma_eq_callback *callback;
334			void *context;
335
336			unsigned int msix_index;
337
338			u32 log2_throttle_limit;
339		} eq;
340
341		struct {
342			gdma_cq_callback *callback;
343			void *context;
344
345			struct gdma_queue *parent; /* For CQ/EQ relationship */
346		} cq;
347	};
348};
349
350struct gdma_queue_spec {
351	enum gdma_queue_type type;
352	bool monitor_avl_buf;
353	unsigned int queue_size;
354
355	/* Extra fields specific to EQ/CQ. */
356	union {
357		struct {
358			gdma_eq_callback *callback;
359			void *context;
360
361			unsigned long log2_throttle_limit;
362			unsigned int msix_index;
363		} eq;
364
365		struct {
366			gdma_cq_callback *callback;
367			void *context;
368
369			struct gdma_queue *parent_eq;
370
371		} cq;
372	};
373};
374
375#define MANA_IRQ_NAME_SZ 32
376
377struct gdma_irq_context {
378	void (*handler)(void *arg);
379	/* Protect the eq_list */
380	spinlock_t lock;
381	struct list_head eq_list;
382	char name[MANA_IRQ_NAME_SZ];
383};
384
385enum gdma_context_flags {
386	GC_PROBE_SUCCEEDED	= 0,
387};
388
389struct gdma_context {
390	struct device		*dev;
391	struct dentry		*mana_pci_debugfs;
392
393	/* Per-vPort max number of queues */
394	unsigned int		max_num_queues;
395	unsigned int		max_num_msix;
396	unsigned int		num_msix_usable;
397	struct xarray		irq_contexts;
398
399	/* L2 MTU */
400	u16 adapter_mtu;
401
402	/* This maps a CQ index to the queue structure. */
403	unsigned int		max_num_cqs;
404	struct gdma_queue	**cq_table;
405
406	/* Protect eq_test_event and test_event_eq_id  */
407	struct mutex		eq_test_event_mutex;
408	struct completion	eq_test_event;
409	u32			test_event_eq_id;
410
411	bool			is_pf;
412	bool			in_service;
413
414	phys_addr_t		bar0_pa;
415	void __iomem		*bar0_va;
416	void __iomem		*shm_base;
417	void __iomem		*db_page_base;
418	phys_addr_t		phys_db_page_base;
419	u32 db_page_size;
420	int                     numa_node;
421
422	/* Shared memory chanenl (used to bootstrap HWC) */
423	struct shm_channel	shm_channel;
424
425	/* Hardware communication channel (HWC) */
426	struct gdma_dev		hwc;
427
428	/* Azure network adapter */
429	struct gdma_dev		mana;
430
431	/* Azure RDMA adapter */
432	struct gdma_dev		mana_ib;
433
434	u64 pf_cap_flags1;
435
436	struct workqueue_struct *service_wq;
437
438	unsigned long		flags;
439};
440
441static inline bool mana_gd_is_mana(struct gdma_dev *gd)
442{
443	return gd->dev_id.type == GDMA_DEVICE_MANA;
444}
445
446static inline bool mana_gd_is_hwc(struct gdma_dev *gd)
447{
448	return gd->dev_id.type == GDMA_DEVICE_HWC;
449}
450
451u8 *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, u32 wqe_offset);
452u32 mana_gd_wq_avail_space(struct gdma_queue *wq);
453
454int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq);
455
456int mana_gd_create_hwc_queue(struct gdma_dev *gd,
457			     const struct gdma_queue_spec *spec,
458			     struct gdma_queue **queue_ptr);
459
460int mana_gd_create_mana_eq(struct gdma_dev *gd,
461			   const struct gdma_queue_spec *spec,
462			   struct gdma_queue **queue_ptr);
463
464int mana_gd_create_mana_wq_cq(struct gdma_dev *gd,
465			      const struct gdma_queue_spec *spec,
466			      struct gdma_queue **queue_ptr);
467
468void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue);
469
470int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe);
471
472void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit);
473
474struct gdma_wqe {
475	u32 reserved	:24;
476	u32 last_vbytes	:8;
477
478	union {
479		u32 flags;
480
481		struct {
482			u32 num_sge		:8;
483			u32 inline_oob_size_div4:3;
484			u32 client_oob_in_sgl	:1;
485			u32 reserved1		:4;
486			u32 client_data_unit	:14;
487			u32 reserved2		:2;
488		};
489	};
490}; /* HW DATA */
491
492#define INLINE_OOB_SMALL_SIZE 8
493#define INLINE_OOB_LARGE_SIZE 24
494
495#define MANA_MAX_TX_WQE_SGL_ENTRIES 30
496
497#define MAX_TX_WQE_SIZE 512
498#define MAX_RX_WQE_SIZE 256
499
500#define MAX_TX_WQE_SGL_ENTRIES	((GDMA_MAX_SQE_SIZE -			   \
501			sizeof(struct gdma_sge) - INLINE_OOB_SMALL_SIZE) / \
502			sizeof(struct gdma_sge))
503
504#define MAX_RX_WQE_SGL_ENTRIES	((GDMA_MAX_RQE_SIZE -			   \
505			sizeof(struct gdma_sge)) / sizeof(struct gdma_sge))
506
507struct gdma_cqe {
508	u32 cqe_data[GDMA_COMP_DATA_SIZE / 4];
509
510	union {
511		u32 as_uint32;
512
513		struct {
514			u32 wq_num	: 24;
515			u32 is_sq	: 1;
516			u32 reserved	: 4;
517			u32 owner_bits	: 3;
518		};
519	} cqe_info;
520}; /* HW DATA */
521
522#define GDMA_CQE_OWNER_BITS 3
523
524#define GDMA_CQE_OWNER_MASK ((1 << GDMA_CQE_OWNER_BITS) - 1)
525
526#define SET_ARM_BIT 1
527
528#define GDMA_EQE_OWNER_BITS 3
529
530union gdma_eqe_info {
531	u32 as_uint32;
532
533	struct {
534		u32 type	: 8;
535		u32 reserved1	: 8;
536		u32 client_id	: 2;
537		u32 reserved2	: 11;
538		u32 owner_bits	: 3;
539	};
540}; /* HW DATA */
541
542#define GDMA_EQE_OWNER_MASK ((1 << GDMA_EQE_OWNER_BITS) - 1)
543#define INITIALIZED_OWNER_BIT(log2_num_entries) (1UL << (log2_num_entries))
544
545struct gdma_eqe {
546	u32 details[GDMA_EVENT_DATA_SIZE / 4];
547	u32 eqe_info;
548}; /* HW DATA */
549
550#define GDMA_REG_DB_PAGE_OFFSET	8
551#define GDMA_REG_DB_PAGE_SIZE	0x10
552#define GDMA_REG_SHM_OFFSET	0x18
553
554#define GDMA_PF_REG_DB_PAGE_SIZE	0xD0
555#define GDMA_PF_REG_DB_PAGE_OFF		0xC8
556#define GDMA_PF_REG_SHM_OFF		0x70
557
558#define GDMA_SRIOV_REG_CFG_BASE_OFF	0x108
559
560#define MANA_PF_DEVICE_ID 0x00B9
561#define MANA_VF_DEVICE_ID 0x00BA
562
563struct gdma_posted_wqe_info {
564	u32 wqe_size_in_bu;
565};
566
567/* GDMA_GENERATE_TEST_EQE */
568struct gdma_generate_test_event_req {
569	struct gdma_req_hdr hdr;
570	u32 queue_index;
571}; /* HW DATA */
572
573/* GDMA_VERIFY_VF_DRIVER_VERSION */
574enum {
575	GDMA_PROTOCOL_V1	= 1,
576	GDMA_PROTOCOL_FIRST	= GDMA_PROTOCOL_V1,
577	GDMA_PROTOCOL_LAST	= GDMA_PROTOCOL_V1,
578};
579
580#define GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT BIT(0)
581
582/* Advertise to the NIC firmware: the NAPI work_done variable race is fixed,
583 * so the driver is able to reliably support features like busy_poll.
584 */
585#define GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX BIT(2)
586#define GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG BIT(3)
587#define GDMA_DRV_CAP_FLAG_1_GDMA_PAGES_4MB_1GB_2GB BIT(4)
588#define GDMA_DRV_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT BIT(5)
589
590/* Driver can handle holes (zeros) in the device list */
591#define GDMA_DRV_CAP_FLAG_1_DEV_LIST_HOLES_SUP BIT(11)
592
593/* Driver supports dynamic MSI-X vector allocation */
594#define GDMA_DRV_CAP_FLAG_1_DYNAMIC_IRQ_ALLOC_SUPPORT BIT(13)
595
596/* Driver can self reset on EQE notification */
597#define GDMA_DRV_CAP_FLAG_1_SELF_RESET_ON_EQE BIT(14)
598
599/* Driver can self reset on FPGA Reconfig EQE notification */
600#define GDMA_DRV_CAP_FLAG_1_HANDLE_RECONFIG_EQE BIT(17)
601#define GDMA_DRV_CAP_FLAG_1_HW_VPORT_LINK_AWARE BIT(6)
602
603/* Driver supports linearizing the skb when num_sge exceeds hardware limit */
604#define GDMA_DRV_CAP_FLAG_1_SKB_LINEARIZE BIT(20)
605
606/* Driver can send HWC periodically to query stats */
607#define GDMA_DRV_CAP_FLAG_1_PERIODIC_STATS_QUERY BIT(21)
608
609/* Driver can handle hardware recovery events during probe */
610#define GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY BIT(22)
611
612#define GDMA_DRV_CAP_FLAGS1 \
613	(GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \
614	 GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX | \
615	 GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG | \
616	 GDMA_DRV_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT | \
617	 GDMA_DRV_CAP_FLAG_1_DEV_LIST_HOLES_SUP | \
618	 GDMA_DRV_CAP_FLAG_1_DYNAMIC_IRQ_ALLOC_SUPPORT | \
619	 GDMA_DRV_CAP_FLAG_1_SELF_RESET_ON_EQE | \
620	 GDMA_DRV_CAP_FLAG_1_HANDLE_RECONFIG_EQE | \
621	 GDMA_DRV_CAP_FLAG_1_HW_VPORT_LINK_AWARE | \
622	 GDMA_DRV_CAP_FLAG_1_PERIODIC_STATS_QUERY | \
623	 GDMA_DRV_CAP_FLAG_1_SKB_LINEARIZE | \
624	 GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY)
625
626#define GDMA_DRV_CAP_FLAGS2 0
627
628#define GDMA_DRV_CAP_FLAGS3 0
629
630#define GDMA_DRV_CAP_FLAGS4 0
631
632struct gdma_verify_ver_req {
633	struct gdma_req_hdr hdr;
634
635	/* Mandatory fields required for protocol establishment */
636	u64 protocol_ver_min;
637	u64 protocol_ver_max;
638
639	/* Gdma Driver Capability Flags */
640	u64 gd_drv_cap_flags1;
641	u64 gd_drv_cap_flags2;
642	u64 gd_drv_cap_flags3;
643	u64 gd_drv_cap_flags4;
644
645	/* Advisory fields */
646	u64 drv_ver;
647	u32 os_type; /* Linux = 0x10; Windows = 0x20; Other = 0x30 */
648	u32 reserved;
649	u32 os_ver_major;
650	u32 os_ver_minor;
651	u32 os_ver_build;
652	u32 os_ver_platform;
653	u64 reserved_2;
654	u8 os_ver_str1[128];
655	u8 os_ver_str2[128];
656	u8 os_ver_str3[128];
657	u8 os_ver_str4[128];
658}; /* HW DATA */
659
660struct gdma_verify_ver_resp {
661	struct gdma_resp_hdr hdr;
662	u64 gdma_protocol_ver;
663	u64 pf_cap_flags1;
664	u64 pf_cap_flags2;
665	u64 pf_cap_flags3;
666	u64 pf_cap_flags4;
667}; /* HW DATA */
668
669/* GDMA_QUERY_MAX_RESOURCES */
670struct gdma_query_max_resources_resp {
671	struct gdma_resp_hdr hdr;
672	u32 status;
673	u32 max_sq;
674	u32 max_rq;
675	u32 max_cq;
676	u32 max_eq;
677	u32 max_db;
678	u32 max_mst;
679	u32 max_cq_mod_ctx;
680	u32 max_mod_cq;
681	u32 max_msix;
682}; /* HW DATA */
683
684/* GDMA_LIST_DEVICES */
685#define GDMA_DEV_LIST_SIZE 64
686struct gdma_list_devices_resp {
687	struct gdma_resp_hdr hdr;
688	u32 num_of_devs;
689	u32 reserved;
690	struct gdma_dev_id devs[GDMA_DEV_LIST_SIZE];
691}; /* HW DATA */
692
693/* GDMA_REGISTER_DEVICE */
694struct gdma_register_device_resp {
695	struct gdma_resp_hdr hdr;
696	u32 pdid;
697	u32 gpa_mkey;
698	u32 db_id;
699}; /* HW DATA */
700
701struct gdma_allocate_resource_range_req {
702	struct gdma_req_hdr hdr;
703	u32 resource_type;
704	u32 num_resources;
705	u32 alignment;
706	u32 allocated_resources;
707};
708
709struct gdma_allocate_resource_range_resp {
710	struct gdma_resp_hdr hdr;
711	u32 allocated_resources;
712};
713
714struct gdma_destroy_resource_range_req {
715	struct gdma_req_hdr hdr;
716	u32 resource_type;
717	u32 num_resources;
718	u32 allocated_resources;
719};
720
721/* GDMA_CREATE_QUEUE */
722struct gdma_create_queue_req {
723	struct gdma_req_hdr hdr;
724	u32 type;
725	u32 reserved1;
726	u32 pdid;
727	u32 doolbell_id;
728	u64 gdma_region;
729	u32 reserved2;
730	u32 queue_size;
731	u32 log2_throttle_limit;
732	u32 eq_pci_msix_index;
733	u32 cq_mod_ctx_id;
734	u32 cq_parent_eq_id;
735	u8  rq_drop_on_overrun;
736	u8  rq_err_on_wqe_overflow;
737	u8  rq_chain_rec_wqes;
738	u8  sq_hw_db;
739	u32 reserved3;
740}; /* HW DATA */
741
742struct gdma_create_queue_resp {
743	struct gdma_resp_hdr hdr;
744	u32 queue_index;
745}; /* HW DATA */
746
747/* GDMA_DISABLE_QUEUE */
748struct gdma_disable_queue_req {
749	struct gdma_req_hdr hdr;
750	u32 type;
751	u32 queue_index;
752	u32 alloc_res_id_on_creation;
753}; /* HW DATA */
754
755/* GDMA_QUERY_HWC_TIMEOUT */
756struct gdma_query_hwc_timeout_req {
757	struct gdma_req_hdr hdr;
758	u32 timeout_ms;
759	u32 reserved;
760};
761
762struct gdma_query_hwc_timeout_resp {
763	struct gdma_resp_hdr hdr;
764	u32 timeout_ms;
765	u32 reserved;
766};
767
768enum gdma_mr_access_flags {
769	GDMA_ACCESS_FLAG_LOCAL_READ = BIT_ULL(0),
770	GDMA_ACCESS_FLAG_LOCAL_WRITE = BIT_ULL(1),
771	GDMA_ACCESS_FLAG_REMOTE_READ = BIT_ULL(2),
772	GDMA_ACCESS_FLAG_REMOTE_WRITE = BIT_ULL(3),
773	GDMA_ACCESS_FLAG_REMOTE_ATOMIC = BIT_ULL(4),
774};
775
776/* GDMA_CREATE_DMA_REGION */
777struct gdma_create_dma_region_req {
778	struct gdma_req_hdr hdr;
779
780	/* The total size of the DMA region */
781	u64 length;
782
783	/* The offset in the first page */
784	u32 offset_in_page;
785
786	/* enum gdma_page_type */
787	u32 gdma_page_type;
788
789	/* The total number of pages */
790	u32 page_count;
791
792	/* If page_addr_list_len is smaller than page_count,
793	 * the remaining page addresses will be added via the
794	 * message GDMA_DMA_REGION_ADD_PAGES.
795	 */
796	u32 page_addr_list_len;
797	u64 page_addr_list[];
798}; /* HW DATA */
799
800struct gdma_create_dma_region_resp {
801	struct gdma_resp_hdr hdr;
802	u64 dma_region_handle;
803}; /* HW DATA */
804
805/* GDMA_DMA_REGION_ADD_PAGES */
806struct gdma_dma_region_add_pages_req {
807	struct gdma_req_hdr hdr;
808
809	u64 dma_region_handle;
810
811	u32 page_addr_list_len;
812	u32 reserved3;
813
814	u64 page_addr_list[];
815}; /* HW DATA */
816
817/* GDMA_DESTROY_DMA_REGION */
818struct gdma_destroy_dma_region_req {
819	struct gdma_req_hdr hdr;
820
821	u64 dma_region_handle;
822}; /* HW DATA */
823
824enum gdma_pd_flags {
825	GDMA_PD_FLAG_INVALID = 0,
826	GDMA_PD_FLAG_ALLOW_GPA_MR = 1,
827};
828
829struct gdma_create_pd_req {
830	struct gdma_req_hdr hdr;
831	enum gdma_pd_flags flags;
832	u32 reserved;
833};/* HW DATA */
834
835struct gdma_create_pd_resp {
836	struct gdma_resp_hdr hdr;
837	u64 pd_handle;
838	u32 pd_id;
839	u32 reserved;
840};/* HW DATA */
841
842struct gdma_destroy_pd_req {
843	struct gdma_req_hdr hdr;
844	u64 pd_handle;
845};/* HW DATA */
846
847struct gdma_destory_pd_resp {
848	struct gdma_resp_hdr hdr;
849};/* HW DATA */
850
851enum gdma_mr_type {
852	/*
853	 * Guest Physical Address - MRs of this type allow access
854	 * to any DMA-mapped memory using bus-logical address
855	 */
856	GDMA_MR_TYPE_GPA = 1,
857	/* Guest Virtual Address - MRs of this type allow access
858	 * to memory mapped by PTEs associated with this MR using a virtual
859	 * address that is set up in the MST
860	 */
861	GDMA_MR_TYPE_GVA = 2,
862	/* Guest zero-based address MRs */
863	GDMA_MR_TYPE_ZBVA = 4,
864};
865
866struct gdma_create_mr_params {
867	u64 pd_handle;
868	enum gdma_mr_type mr_type;
869	union {
870		struct {
871			u64 dma_region_handle;
872			u64 virtual_address;
873			enum gdma_mr_access_flags access_flags;
874		} gva;
875		struct {
876			u64 dma_region_handle;
877			enum gdma_mr_access_flags access_flags;
878		} zbva;
879	};
880};
881
882struct gdma_create_mr_request {
883	struct gdma_req_hdr hdr;
884	u64 pd_handle;
885	enum gdma_mr_type mr_type;
886	u32 reserved_1;
887
888	union {
889		struct {
890			u64 dma_region_handle;
891			u64 virtual_address;
892			enum gdma_mr_access_flags access_flags;
893		} gva;
894		struct {
895			u64 dma_region_handle;
896			enum gdma_mr_access_flags access_flags;
897		} zbva;
898	};
899	u32 reserved_2;
900};/* HW DATA */
901
902struct gdma_create_mr_response {
903	struct gdma_resp_hdr hdr;
904	u64 mr_handle;
905	u32 lkey;
906	u32 rkey;
907};/* HW DATA */
908
909struct gdma_destroy_mr_request {
910	struct gdma_req_hdr hdr;
911	u64 mr_handle;
912};/* HW DATA */
913
914struct gdma_destroy_mr_response {
915	struct gdma_resp_hdr hdr;
916};/* HW DATA */
917
918int mana_gd_verify_vf_version(struct pci_dev *pdev);
919
920int mana_gd_register_device(struct gdma_dev *gd);
921int mana_gd_deregister_device(struct gdma_dev *gd);
922
923int mana_gd_post_work_request(struct gdma_queue *wq,
924			      const struct gdma_wqe_request *wqe_req,
925			      struct gdma_posted_wqe_info *wqe_info);
926
927int mana_gd_post_and_ring(struct gdma_queue *queue,
928			  const struct gdma_wqe_request *wqe,
929			  struct gdma_posted_wqe_info *wqe_info);
930
931int mana_gd_alloc_res_map(u32 res_avail, struct gdma_resource *r);
932void mana_gd_free_res_map(struct gdma_resource *r);
933
934void mana_gd_wq_ring_doorbell(struct gdma_context *gc,
935			      struct gdma_queue *queue);
936
937int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,
938			 struct gdma_mem_info *gmi);
939
940void mana_gd_free_memory(struct gdma_mem_info *gmi);
941
942int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req,
943			 u32 resp_len, void *resp);
944
945int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle);
946void mana_register_debugfs(void);
947void mana_unregister_debugfs(void);
948
949int mana_rdma_service_event(struct gdma_context *gc, enum gdma_service_type event);
950
951int mana_gd_suspend(struct pci_dev *pdev, pm_message_t state);
952int mana_gd_resume(struct pci_dev *pdev);
953
954bool mana_need_log(struct gdma_context *gc, int err);
955
956#endif /* _GDMA_H */