cmd.h

  1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
  2/*
  3 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  4 */
  5
  6#ifndef MLX5_VFIO_CMD_H
  7#define MLX5_VFIO_CMD_H
  8
  9#include <linux/kernel.h>
 10#include <linux/vfio_pci_core.h>
 11#include <linux/mlx5/driver.h>
 12#include <linux/mlx5/vport.h>
 13#include <linux/mlx5/cq.h>
 14#include <linux/mlx5/qp.h>
 15
 16enum mlx5_vf_migf_state {
 17	MLX5_MIGF_STATE_ERROR = 1,
 18	MLX5_MIGF_STATE_PRE_COPY_ERROR,
 19	MLX5_MIGF_STATE_PRE_COPY,
 20	MLX5_MIGF_STATE_SAVE_STOP_COPY_CHUNK,
 21	MLX5_MIGF_STATE_COMPLETE,
 22};
 23
 24enum mlx5_vf_load_state {
 25	MLX5_VF_LOAD_STATE_READ_HEADER,
 26	MLX5_VF_LOAD_STATE_PREP_HEADER_DATA,
 27	MLX5_VF_LOAD_STATE_READ_HEADER_DATA,
 28	MLX5_VF_LOAD_STATE_PREP_IMAGE,
 29	MLX5_VF_LOAD_STATE_READ_IMAGE,
 30	MLX5_VF_LOAD_STATE_LOAD_IMAGE,
 31};
 32
 33struct mlx5_vf_migration_tag_stop_copy_data {
 34	__le64 stop_copy_size;
 35};
 36
 37enum mlx5_vf_migf_header_flags {
 38	MLX5_MIGF_HEADER_FLAGS_TAG_MANDATORY = 0,
 39	MLX5_MIGF_HEADER_FLAGS_TAG_OPTIONAL = 1 << 0,
 40};
 41
 42enum mlx5_vf_migf_header_tag {
 43	MLX5_MIGF_HEADER_TAG_FW_DATA = 0,
 44	MLX5_MIGF_HEADER_TAG_STOP_COPY_SIZE = 1 << 0,
 45};
 46
 47struct mlx5_vf_migration_header {
 48	__le64 record_size;
 49	/* For future use in case we may need to change the kernel protocol */
 50	__le32 flags; /* Use mlx5_vf_migf_header_flags */
 51	__le32 tag; /* Use mlx5_vf_migf_header_tag */
 52	__u8 data[]; /* Its size is given in the record_size */
 53};
 54
 55struct mlx5_vhca_data_buffer {
 56	struct page **page_list;
 57	struct dma_iova_state state;
 58	loff_t start_pos;
 59	u64 length;
 60	u32 npages;
 61	u32 mkey;
 62	u32 *mkey_in;
 63	enum dma_data_direction dma_dir;
 64	u8 stop_copy_chunk_num;
 65	struct list_head buf_elm;
 66	struct mlx5_vf_migration_file *migf;
 67};
 68
 69struct mlx5vf_async_data {
 70	struct mlx5_async_work cb_work;
 71	struct work_struct work;
 72	struct mlx5_vhca_data_buffer *buf;
 73	struct mlx5_vhca_data_buffer *header_buf;
 74	int status;
 75	u8 stop_copy_chunk:1;
 76	void *out;
 77};
 78
 79struct mlx5vf_save_work_data {
 80	struct mlx5_vf_migration_file *migf;
 81	size_t next_required_umem_size;
 82	struct work_struct work;
 83	u8 chunk_num;
 84};
 85
 86#define MAX_NUM_CHUNKS 2
 87
 88struct mlx5_vf_migration_file {
 89	struct file *filp;
 90	struct mutex lock;
 91	enum mlx5_vf_migf_state state;
 92
 93	enum mlx5_vf_load_state load_state;
 94	u32 pdn;
 95	loff_t max_pos;
 96	u64 record_size;
 97	u32 record_tag;
 98	u64 stop_copy_prep_size;
 99	u64 pre_copy_initial_bytes;
100	size_t next_required_umem_size;
101	u8 num_ready_chunks;
102	/* Upon chunk mode preserve another set of buffers for stop_copy phase */
103	struct mlx5_vhca_data_buffer *buf[MAX_NUM_CHUNKS];
104	struct mlx5_vhca_data_buffer *buf_header[MAX_NUM_CHUNKS];
105	struct mlx5vf_save_work_data save_data[MAX_NUM_CHUNKS];
106	spinlock_t list_lock;
107	struct list_head buf_list;
108	struct list_head avail_list;
109	struct mlx5vf_pci_core_device *mvdev;
110	wait_queue_head_t poll_wait;
111	struct completion save_comp;
112	struct mlx5_async_ctx async_ctx;
113	struct mlx5vf_async_data async_data;
114};
115
116struct mlx5_vhca_cq_buf {
117	struct mlx5_frag_buf_ctrl fbc;
118	struct mlx5_frag_buf frag_buf;
119	int cqe_size;
120	int nent;
121};
122
123struct mlx5_vhca_cq {
124	struct mlx5_vhca_cq_buf buf;
125	struct mlx5_db db;
126	struct mlx5_core_cq mcq;
127	size_t ncqe;
128};
129
130struct mlx5_vhca_recv_buf {
131	u32 npages;
132	struct page **page_list;
133	struct dma_iova_state state;
134	u32 next_rq_offset;
135	u32 *mkey_in;
136	u32 mkey;
137};
138
139struct mlx5_vhca_qp {
140	struct mlx5_frag_buf buf;
141	struct mlx5_db db;
142	struct mlx5_vhca_recv_buf recv_buf;
143	u32 tracked_page_size;
144	u32 max_msg_size;
145	u32 qpn;
146	struct {
147		unsigned int pc;
148		unsigned int cc;
149		unsigned int wqe_cnt;
150		__be32 *db;
151		struct mlx5_frag_buf_ctrl fbc;
152	} rq;
153};
154
155struct mlx5_vhca_page_tracker {
156	u32 id;
157	u32 pdn;
158	u8 is_err:1;
159	u8 object_changed:1;
160	struct mlx5_uars_page *uar;
161	struct mlx5_vhca_cq cq;
162	struct mlx5_vhca_qp *host_qp;
163	struct mlx5_vhca_qp *fw_qp;
164	struct mlx5_nb nb;
165	int status;
166};
167
168struct mlx5vf_pci_core_device {
169	struct vfio_pci_core_device core_device;
170	int vf_id;
171	u16 vhca_id;
172	u8 migrate_cap:1;
173	u8 deferred_reset:1;
174	u8 mdev_detach:1;
175	u8 log_active:1;
176	u8 chunk_mode:1;
177	struct completion tracker_comp;
178	/* protect migration state */
179	struct mutex state_mutex;
180	enum vfio_device_mig_state mig_state;
181	/* protect the reset_done flow */
182	spinlock_t reset_lock;
183	struct mlx5_vf_migration_file *resuming_migf;
184	struct mlx5_vf_migration_file *saving_migf;
185	struct mlx5_vhca_page_tracker tracker;
186	struct workqueue_struct *cb_wq;
187	struct notifier_block nb;
188	struct mlx5_core_dev *mdev;
189};
190
191enum {
192	MLX5VF_QUERY_INC = (1UL << 0),
193	MLX5VF_QUERY_FINAL = (1UL << 1),
194	MLX5VF_QUERY_CLEANUP = (1UL << 2),
195};
196
197int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod);
198int mlx5vf_cmd_resume_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod);
199int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev,
200					  size_t *state_size, u64 *total_size,
201					  u8 query_flags);
202void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev,
203			       const struct vfio_migration_ops *mig_ops,
204			       const struct vfio_log_ops *log_ops);
205void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev);
206void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev);
207int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev,
208			       struct mlx5_vf_migration_file *migf,
209			       struct mlx5_vhca_data_buffer *buf, bool inc,
210			       bool track);
211int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev,
212			       struct mlx5_vf_migration_file *migf,
213			       struct mlx5_vhca_data_buffer *buf);
214int mlx5vf_cmd_alloc_pd(struct mlx5_vf_migration_file *migf);
215void mlx5vf_cmd_dealloc_pd(struct mlx5_vf_migration_file *migf);
216void mlx5fv_cmd_clean_migf_resources(struct mlx5_vf_migration_file *migf);
217struct mlx5_vhca_data_buffer *
218mlx5vf_alloc_data_buffer(struct mlx5_vf_migration_file *migf, u32 npages,
219			 enum dma_data_direction dma_dir);
220void mlx5vf_free_data_buffer(struct mlx5_vhca_data_buffer *buf);
221struct mlx5_vhca_data_buffer *
222mlx5vf_get_data_buffer(struct mlx5_vf_migration_file *migf, u32 npages,
223		       enum dma_data_direction dma_dir);
224void mlx5vf_put_data_buffer(struct mlx5_vhca_data_buffer *buf);
225static inline struct page *
226mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer *buf,
227			  unsigned long offset)
228{
229	int page_entry = offset / PAGE_SIZE;
230
231	if (page_entry >= buf->npages)
232		return NULL;
233
234	return buf->page_list[page_entry];
235}
236void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev);
237void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev,
238			enum mlx5_vf_migf_state *last_save_state);
239void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work);
240void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf,
241				   u8 chunk_num, size_t next_required_umem_size);
242int mlx5vf_start_page_tracker(struct vfio_device *vdev,
243		struct rb_root_cached *ranges, u32 nnodes, u64 *page_size);
244int mlx5vf_stop_page_tracker(struct vfio_device *vdev);
245int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova,
246			unsigned long length, struct iova_bitmap *dirty);
247#endif /* MLX5_VFIO_CMD_H */