octep_vdpa_hw.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright (C) 2024 Marvell. */
  3
  4#include <linux/iopoll.h>
  5#include <linux/build_bug.h>
  6
  7#include "octep_vdpa.h"
  8
  9enum octep_mbox_ids {
 10	OCTEP_MBOX_MSG_SET_VQ_STATE = 1,
 11	OCTEP_MBOX_MSG_GET_VQ_STATE,
 12};
 13
 14#define OCTEP_HW_TIMEOUT       10000000
 15
 16#define MBOX_OFFSET            64
 17#define MBOX_RSP_MASK          0x00000001
 18#define MBOX_RC_MASK           0x0000FFFE
 19
 20#define MBOX_RSP_TO_ERR(val)   (-(((val) & MBOX_RC_MASK) >> 2))
 21#define MBOX_AVAIL(val)        (((val) & MBOX_RSP_MASK))
 22#define MBOX_RSP(val)          ((val) & (MBOX_RC_MASK | MBOX_RSP_MASK))
 23
 24#define DEV_RST_ACK_BIT        7
 25#define FEATURE_SEL_ACK_BIT    15
 26#define QUEUE_SEL_ACK_BIT      15
 27
 28struct octep_mbox_hdr {
 29	u8 ver;
 30	u8 rsvd1;
 31	u16 id;
 32	u16 rsvd2;
 33#define MBOX_REQ_SIG (0xdead)
 34#define MBOX_RSP_SIG (0xbeef)
 35	u16 sig;
 36};
 37
 38struct octep_mbox_sts {
 39	u16 rsp:1;
 40	u16 rc:15;
 41	u16 rsvd;
 42};
 43
 44struct octep_mbox {
 45	struct octep_mbox_hdr hdr;
 46	struct octep_mbox_sts sts;
 47	u64 rsvd;
 48	u32 data[];
 49};
 50
 51static inline struct octep_mbox __iomem *octep_get_mbox(struct octep_hw *oct_hw)
 52{
 53	return (struct octep_mbox __iomem *)(oct_hw->dev_cfg + MBOX_OFFSET);
 54}
 55
 56static inline int octep_wait_for_mbox_avail(struct octep_mbox __iomem *mbox)
 57{
 58	u32 val;
 59
 60	return readx_poll_timeout(ioread32, &mbox->sts, val, MBOX_AVAIL(val), 10,
 61				  OCTEP_HW_TIMEOUT);
 62}
 63
 64static inline int octep_wait_for_mbox_rsp(struct octep_mbox __iomem *mbox)
 65{
 66	u32 val;
 67
 68	return readx_poll_timeout(ioread32, &mbox->sts, val, MBOX_RSP(val), 10,
 69				  OCTEP_HW_TIMEOUT);
 70}
 71
 72static inline void octep_write_hdr(struct octep_mbox __iomem *mbox, u16 id, u16 sig)
 73{
 74	iowrite16(id, &mbox->hdr.id);
 75	iowrite16(sig, &mbox->hdr.sig);
 76}
 77
 78static inline u32 octep_read_sig(struct octep_mbox __iomem *mbox)
 79{
 80	return ioread16(&mbox->hdr.sig);
 81}
 82
 83static inline void octep_write_sts(struct octep_mbox __iomem *mbox, u32 sts)
 84{
 85	iowrite32(sts, &mbox->sts);
 86}
 87
 88static inline u32 octep_read_sts(struct octep_mbox __iomem *mbox)
 89{
 90	return ioread32(&mbox->sts);
 91}
 92
 93static inline u32 octep_read32_word(struct octep_mbox __iomem *mbox, u16 word_idx)
 94{
 95	return ioread32(&mbox->data[word_idx]);
 96}
 97
 98static inline void octep_write32_word(struct octep_mbox __iomem *mbox, u16 word_idx, u32 word)
 99{
100	return iowrite32(word, &mbox->data[word_idx]);
101}
102
103static int octep_process_mbox(struct octep_hw *oct_hw, u16 id, u16 qid, void *buffer,
104			      u32 buf_size, bool write)
105{
106	struct octep_mbox __iomem *mbox = octep_get_mbox(oct_hw);
107	struct pci_dev *pdev = oct_hw->pdev;
108	u32 *p = (u32 *)buffer;
109	u16 data_wds;
110	int ret, i;
111	u32 val;
112
113	if (!IS_ALIGNED(buf_size, 4))
114		return -EINVAL;
115
116	/* Make sure mbox space is available */
117	ret = octep_wait_for_mbox_avail(mbox);
118	if (ret) {
119		dev_warn(&pdev->dev, "Timeout waiting for previous mbox data to be consumed\n");
120		return ret;
121	}
122	data_wds = buf_size / 4;
123
124	if (write) {
125		for (i = 1; i <= data_wds; i++) {
126			octep_write32_word(mbox, i, *p);
127			p++;
128		}
129	}
130	octep_write32_word(mbox, 0, (u32)qid);
131	octep_write_sts(mbox, 0);
132
133	octep_write_hdr(mbox, id, MBOX_REQ_SIG);
134
135	ret = octep_wait_for_mbox_rsp(mbox);
136	if (ret) {
137		dev_warn(&pdev->dev, "Timeout waiting for mbox : %d response\n", id);
138		return ret;
139	}
140
141	val = octep_read_sig(mbox);
142	if ((val & 0xFFFF) != MBOX_RSP_SIG) {
143		dev_warn(&pdev->dev, "Invalid Signature from mbox : %d response\n", id);
144		return -EINVAL;
145	}
146
147	val = octep_read_sts(mbox);
148	if (val & MBOX_RC_MASK) {
149		ret = MBOX_RSP_TO_ERR(val);
150		dev_warn(&pdev->dev, "Error while processing mbox : %d, err %d\n", id, ret);
151		return ret;
152	}
153
154	if (!write)
155		for (i = 1; i <= data_wds; i++)
156			*p++ = octep_read32_word(mbox, i);
157
158	return 0;
159}
160
161static void octep_mbox_init(struct octep_mbox __iomem *mbox)
162{
163	iowrite32(1, &mbox->sts);
164}
165
166int octep_verify_features(u64 features)
167{
168	/* Minimum features to expect */
169	if (!(features & BIT_ULL(VIRTIO_F_VERSION_1)))
170		return -EOPNOTSUPP;
171
172	if (!(features & BIT_ULL(VIRTIO_F_NOTIFICATION_DATA)))
173		return -EOPNOTSUPP;
174
175	if (!(features & BIT_ULL(VIRTIO_F_RING_PACKED)))
176		return -EOPNOTSUPP;
177
178	return 0;
179}
180
181u8 octep_hw_get_status(struct octep_hw *oct_hw)
182{
183	return ioread8(&oct_hw->common_cfg->device_status);
184}
185
186void octep_hw_set_status(struct octep_hw *oct_hw, u8 status)
187{
188	iowrite8(status, &oct_hw->common_cfg->device_status);
189}
190
191void octep_hw_reset(struct octep_hw *oct_hw)
192{
193	u8 val;
194
195	octep_hw_set_status(oct_hw, 0 | BIT(DEV_RST_ACK_BIT));
196	if (readx_poll_timeout(ioread8, &oct_hw->common_cfg->device_status, val, !val, 10,
197			       OCTEP_HW_TIMEOUT)) {
198		dev_warn(&oct_hw->pdev->dev, "Octeon device reset timeout\n");
199		return;
200	}
201}
202
203static int feature_sel_write_with_timeout(struct octep_hw *oct_hw, u32 select, void __iomem *addr)
204{
205	u32 val;
206
207	iowrite32(select | BIT(FEATURE_SEL_ACK_BIT), addr);
208
209	if (readx_poll_timeout(ioread32, addr, val, val == select, 10, OCTEP_HW_TIMEOUT)) {
210		dev_warn(&oct_hw->pdev->dev, "Feature select%d write timeout\n", select);
211		return -1;
212	}
213	return 0;
214}
215
216u64 octep_hw_get_dev_features(struct octep_hw *oct_hw)
217{
218	u32 features_lo, features_hi;
219
220	if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->device_feature_select))
221		return 0;
222
223	features_lo = ioread32(&oct_hw->common_cfg->device_feature);
224
225	if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->device_feature_select))
226		return 0;
227
228	features_hi = ioread32(&oct_hw->common_cfg->device_feature);
229
230	return ((u64)features_hi << 32) | features_lo;
231}
232
233u64 octep_hw_get_drv_features(struct octep_hw *oct_hw)
234{
235	u32 features_lo, features_hi;
236
237	if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->guest_feature_select))
238		return 0;
239
240	features_lo = ioread32(&oct_hw->common_cfg->guest_feature);
241
242	if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->guest_feature_select))
243		return 0;
244
245	features_hi = ioread32(&oct_hw->common_cfg->guest_feature);
246
247	return ((u64)features_hi << 32) | features_lo;
248}
249
250void octep_hw_set_drv_features(struct octep_hw *oct_hw, u64 features)
251{
252	if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->guest_feature_select))
253		return;
254
255	iowrite32(features & (BIT_ULL(32) - 1), &oct_hw->common_cfg->guest_feature);
256
257	if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->guest_feature_select))
258		return;
259
260	iowrite32(features >> 32, &oct_hw->common_cfg->guest_feature);
261}
262
263void octep_write_queue_select(struct octep_hw *oct_hw, u16 queue_id)
264{
265	u16 val;
266
267	iowrite16(queue_id | BIT(QUEUE_SEL_ACK_BIT), &oct_hw->common_cfg->queue_select);
268
269	if (readx_poll_timeout(ioread16, &oct_hw->common_cfg->queue_select, val, val == queue_id,
270			       10, OCTEP_HW_TIMEOUT)) {
271		dev_warn(&oct_hw->pdev->dev, "Queue select write timeout\n");
272		return;
273	}
274}
275
276void octep_notify_queue(struct octep_hw *oct_hw, u16 qid)
277{
278	iowrite16(qid, oct_hw->vqs[qid].notify_addr);
279}
280
281void octep_read_dev_config(struct octep_hw *oct_hw, u64 offset, void *dst, int length)
282{
283	u8 old_gen, new_gen, *p;
284	int i;
285
286	if (WARN_ON(offset + length > oct_hw->config_size))
287		return;
288
289	do {
290		old_gen = ioread8(&oct_hw->common_cfg->config_generation);
291		p = dst;
292		for (i = 0; i < length; i++)
293			*p++ = ioread8(oct_hw->dev_cfg + offset + i);
294
295		new_gen = ioread8(&oct_hw->common_cfg->config_generation);
296	} while (old_gen != new_gen);
297}
298
299int octep_set_vq_address(struct octep_hw *oct_hw, u16 qid, u64 desc_area, u64 driver_area,
300			 u64 device_area)
301{
302	struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;
303
304	octep_write_queue_select(oct_hw, qid);
305	vp_iowrite64_twopart(desc_area, &cfg->queue_desc_lo,
306			     &cfg->queue_desc_hi);
307	vp_iowrite64_twopart(driver_area, &cfg->queue_avail_lo,
308			     &cfg->queue_avail_hi);
309	vp_iowrite64_twopart(device_area, &cfg->queue_used_lo,
310			     &cfg->queue_used_hi);
311
312	return 0;
313}
314
315int octep_get_vq_state(struct octep_hw *oct_hw, u16 qid, struct vdpa_vq_state *state)
316{
317	return octep_process_mbox(oct_hw, OCTEP_MBOX_MSG_GET_VQ_STATE, qid, state,
318				  sizeof(*state), 0);
319}
320
321int octep_set_vq_state(struct octep_hw *oct_hw, u16 qid, const struct vdpa_vq_state *state)
322{
323	struct vdpa_vq_state q_state;
324
325	memcpy(&q_state, state, sizeof(struct vdpa_vq_state));
326	return octep_process_mbox(oct_hw, OCTEP_MBOX_MSG_SET_VQ_STATE, qid, &q_state,
327				  sizeof(*state), 1);
328}
329
330void octep_set_vq_num(struct octep_hw *oct_hw, u16 qid, u32 num)
331{
332	struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;
333
334	octep_write_queue_select(oct_hw, qid);
335	iowrite16(num, &cfg->queue_size);
336}
337
338void octep_set_vq_ready(struct octep_hw *oct_hw, u16 qid, bool ready)
339{
340	struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;
341
342	octep_write_queue_select(oct_hw, qid);
343	iowrite16(ready, &cfg->queue_enable);
344}
345
346bool octep_get_vq_ready(struct octep_hw *oct_hw, u16 qid)
347{
348	struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;
349
350	octep_write_queue_select(oct_hw, qid);
351	return ioread16(&cfg->queue_enable);
352}
353
354u16 octep_get_vq_size(struct octep_hw *oct_hw)
355{
356	octep_write_queue_select(oct_hw, 0);
357	return ioread16(&oct_hw->common_cfg->queue_size);
358}
359
360static u32 octep_get_config_size(struct octep_hw *oct_hw)
361{
362	switch (oct_hw->dev_id) {
363	case VIRTIO_ID_NET:
364		return sizeof(struct virtio_net_config);
365	case VIRTIO_ID_CRYPTO:
366		return sizeof(struct virtio_crypto_config);
367	default:
368		return 0;
369	}
370}
371
372static void __iomem *octep_get_cap_addr(struct octep_hw *oct_hw, struct virtio_pci_cap *cap)
373{
374	struct device *dev = &oct_hw->pdev->dev;
375	u32 length = le32_to_cpu(cap->length);
376	u32 offset = le32_to_cpu(cap->offset);
377	u8  bar    = cap->bar;
378	u32 len;
379
380	if (bar != OCTEP_HW_CAPS_BAR) {
381		dev_err(dev, "Invalid bar: %u\n", bar);
382		return NULL;
383	}
384	if (offset + length < offset) {
385		dev_err(dev, "offset(%u) + length(%u) overflows\n",
386			offset, length);
387		return NULL;
388	}
389	len = pci_resource_len(oct_hw->pdev, bar);
390	if (offset + length > len) {
391		dev_err(dev, "invalid cap: overflows bar space: %u > %u\n",
392			offset + length, len);
393		return NULL;
394	}
395	return oct_hw->base[bar] + offset;
396}
397
398/* In Octeon DPU device, the virtio config space is completely
399 * emulated by the device's firmware. So, the standard pci config
400 * read apis can't be used for reading the virtio capability.
401 */
402static void octep_pci_caps_read(struct octep_hw *oct_hw, void *buf, size_t len, off_t offset)
403{
404	u8 __iomem *bar = oct_hw->base[OCTEP_HW_CAPS_BAR];
405	u8 *p = buf;
406	size_t i;
407
408	for (i = 0; i < len; i++)
409		*p++ = ioread8(bar + offset + i);
410}
411
412static int octep_pci_signature_verify(struct octep_hw *oct_hw)
413{
414	u32 signature[2];
415
416	octep_pci_caps_read(oct_hw, &signature, sizeof(signature), 0);
417
418	if (signature[0] != OCTEP_FW_READY_SIGNATURE0)
419		return -1;
420
421	if (signature[1] != OCTEP_FW_READY_SIGNATURE1)
422		return -1;
423
424	return 0;
425}
426
427static void octep_vndr_data_process(struct octep_hw *oct_hw,
428				    struct octep_pci_vndr_data *vndr_data)
429{
430	BUILD_BUG_ON(sizeof(struct octep_pci_vndr_data) % 4 != 0);
431
432	switch (vndr_data->id) {
433	case OCTEP_PCI_VNDR_CFG_TYPE_VIRTIO_ID:
434		oct_hw->dev_id = (u8)vndr_data->data;
435		break;
436	default:
437		dev_err(&oct_hw->pdev->dev, "Invalid vendor data id %u\n",
438			vndr_data->id);
439		break;
440	}
441}
442
443int octep_hw_caps_read(struct octep_hw *oct_hw, struct pci_dev *pdev)
444{
445	struct octep_pci_vndr_data vndr_data;
446	struct octep_mbox __iomem *mbox;
447	struct device *dev = &pdev->dev;
448	struct virtio_pci_cap cap;
449	u16 notify_off;
450	int i, ret;
451	u8 pos;
452
453	oct_hw->pdev = pdev;
454	ret = octep_pci_signature_verify(oct_hw);
455	if (ret) {
456		dev_err(dev, "Octeon Virtio FW is not initialized\n");
457		return -EIO;
458	}
459
460	octep_pci_caps_read(oct_hw, &pos, 1, PCI_CAPABILITY_LIST);
461
462	while (pos) {
463		octep_pci_caps_read(oct_hw, &cap, 2, pos);
464
465		if (cap.cap_vndr != PCI_CAP_ID_VNDR) {
466			dev_err(dev, "Found invalid capability vndr id: %d\n", cap.cap_vndr);
467			break;
468		}
469
470		octep_pci_caps_read(oct_hw, &cap, sizeof(cap), pos);
471
472		dev_info(dev, "[%2x] cfg type: %u, bar: %u, offset: %04x, len: %u\n",
473			 pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
474
475		switch (cap.cfg_type) {
476		case VIRTIO_PCI_CAP_COMMON_CFG:
477			oct_hw->common_cfg = octep_get_cap_addr(oct_hw, &cap);
478			break;
479		case VIRTIO_PCI_CAP_NOTIFY_CFG:
480			octep_pci_caps_read(oct_hw, &oct_hw->notify_off_multiplier,
481					    4, pos + sizeof(cap));
482
483			oct_hw->notify_base = octep_get_cap_addr(oct_hw, &cap);
484			oct_hw->notify_bar = cap.bar;
485			oct_hw->notify_base_pa = pci_resource_start(pdev, cap.bar) +
486						 le32_to_cpu(cap.offset);
487			break;
488		case VIRTIO_PCI_CAP_DEVICE_CFG:
489			oct_hw->dev_cfg = octep_get_cap_addr(oct_hw, &cap);
490			break;
491		case VIRTIO_PCI_CAP_ISR_CFG:
492			oct_hw->isr = octep_get_cap_addr(oct_hw, &cap);
493			break;
494		case VIRTIO_PCI_CAP_VENDOR_CFG:
495			octep_pci_caps_read(oct_hw, &vndr_data, sizeof(vndr_data), pos);
496			if (vndr_data.hdr.vendor_id != PCI_VENDOR_ID_CAVIUM) {
497				dev_err(dev, "Invalid vendor data\n");
498				return -EINVAL;
499			}
500
501			octep_vndr_data_process(oct_hw, &vndr_data);
502			break;
503		}
504
505		pos = cap.cap_next;
506	}
507	if (!oct_hw->common_cfg || !oct_hw->notify_base ||
508	    !oct_hw->dev_cfg    || !oct_hw->isr) {
509		dev_err(dev, "Incomplete PCI capabilities");
510		return -EIO;
511	}
512	dev_info(dev, "common cfg mapped at: %p\n", oct_hw->common_cfg);
513	dev_info(dev, "device cfg mapped at: %p\n", oct_hw->dev_cfg);
514	dev_info(dev, "isr cfg mapped at: %p\n", oct_hw->isr);
515	dev_info(dev, "notify base: %p, notify off multiplier: %u\n",
516		 oct_hw->notify_base, oct_hw->notify_off_multiplier);
517
518	oct_hw->config_size = octep_get_config_size(oct_hw);
519	oct_hw->features = octep_hw_get_dev_features(oct_hw);
520
521	ret = octep_verify_features(oct_hw->features);
522	if (ret) {
523		dev_err(&pdev->dev, "Couldn't read features from the device FW\n");
524		return ret;
525	}
526	oct_hw->nr_vring = vp_ioread16(&oct_hw->common_cfg->num_queues);
527
528	oct_hw->vqs = devm_kcalloc(&pdev->dev, oct_hw->nr_vring, sizeof(*oct_hw->vqs), GFP_KERNEL);
529	if (!oct_hw->vqs)
530		return -ENOMEM;
531
532	dev_info(&pdev->dev, "Device features : %llx\n", oct_hw->features);
533	dev_info(&pdev->dev, "Maximum queues : %u\n", oct_hw->nr_vring);
534
535	for (i = 0; i < oct_hw->nr_vring; i++) {
536		octep_write_queue_select(oct_hw, i);
537		notify_off = vp_ioread16(&oct_hw->common_cfg->queue_notify_off);
538		oct_hw->vqs[i].notify_addr = oct_hw->notify_base +
539			notify_off * oct_hw->notify_off_multiplier;
540		oct_hw->vqs[i].cb_notify_addr = (u32 __iomem *)oct_hw->vqs[i].notify_addr + 1;
541		oct_hw->vqs[i].notify_pa = oct_hw->notify_base_pa +
542			notify_off * oct_hw->notify_off_multiplier;
543	}
544	mbox = octep_get_mbox(oct_hw);
545	octep_mbox_init(mbox);
546	dev_info(dev, "mbox mapped at: %p\n", mbox);
547
548	return 0;
549}