idxd.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
  3#ifndef _IDXD_H_
  4#define _IDXD_H_
  5
  6#include <linux/sbitmap.h>
  7#include <linux/dmaengine.h>
  8#include <linux/percpu-rwsem.h>
  9#include <linux/wait.h>
 10#include <linux/cdev.h>
 11#include <linux/idr.h>
 12#include <linux/pci.h>
 13#include <linux/bitmap.h>
 14#include <linux/perf_event.h>
 15#include <linux/iommu.h>
 16#include <linux/crypto.h>
 17#include <uapi/linux/idxd.h>
 18#include "registers.h"
 19
 20#define IDXD_DRIVER_VERSION	"1.00"
 21
 22extern bool tc_override;
 23
 24struct idxd_wq;
 25struct idxd_dev;
 26
 27enum idxd_dev_type {
 28	IDXD_DEV_NONE = -1,
 29	IDXD_DEV_DSA = 0,
 30	IDXD_DEV_IAX,
 31	IDXD_DEV_WQ,
 32	IDXD_DEV_GROUP,
 33	IDXD_DEV_ENGINE,
 34	IDXD_DEV_CDEV,
 35	IDXD_DEV_CDEV_FILE,
 36	IDXD_DEV_MAX_TYPE,
 37};
 38
 39struct idxd_dev {
 40	struct device conf_dev;
 41	enum idxd_dev_type type;
 42};
 43
 44#define IDXD_REG_TIMEOUT	50
 45#define IDXD_DRAIN_TIMEOUT	5000
 46
 47enum idxd_type {
 48	IDXD_TYPE_UNKNOWN = -1,
 49	IDXD_TYPE_DSA = 0,
 50	IDXD_TYPE_IAX,
 51	IDXD_TYPE_MAX,
 52};
 53
 54#define IDXD_NAME_SIZE		128
 55#define IDXD_PMU_EVENT_MAX	64
 56
 57#define IDXD_ENQCMDS_RETRIES		32
 58#define IDXD_ENQCMDS_MAX_RETRIES	64
 59
 60enum idxd_complete_type {
 61	IDXD_COMPLETE_NORMAL = 0,
 62	IDXD_COMPLETE_ABORT,
 63	IDXD_COMPLETE_DEV_FAIL,
 64};
 65
 66struct idxd_desc;
 67
 68struct idxd_device_driver {
 69	const char *name;
 70	enum idxd_dev_type *type;
 71	int (*probe)(struct idxd_dev *idxd_dev);
 72	void (*remove)(struct idxd_dev *idxd_dev);
 73	void (*desc_complete)(struct idxd_desc *desc,
 74			      enum idxd_complete_type comp_type,
 75			      bool free_desc,
 76			      void *ctx, u32 *status);
 77	struct device_driver drv;
 78};
 79
 80extern struct idxd_device_driver dsa_drv;
 81extern struct idxd_device_driver idxd_drv;
 82extern struct idxd_device_driver idxd_dmaengine_drv;
 83extern struct idxd_device_driver idxd_user_drv;
 84
 85#define INVALID_INT_HANDLE	-1
 86struct idxd_irq_entry {
 87	int id;
 88	int vector;
 89	struct llist_head pending_llist;
 90	struct list_head work_list;
 91	/*
 92	 * Lock to protect access between irq thread process descriptor
 93	 * and irq thread processing error descriptor.
 94	 */
 95	spinlock_t list_lock;
 96	int int_handle;
 97	ioasid_t pasid;
 98};
 99
100struct idxd_group {
101	struct idxd_dev idxd_dev;
102	struct idxd_device *idxd;
103	struct grpcfg grpcfg;
104	int id;
105	int num_engines;
106	int num_wqs;
107	bool use_rdbuf_limit;
108	u8 rdbufs_allowed;
109	u8 rdbufs_reserved;
110	int tc_a;
111	int tc_b;
112	int desc_progress_limit;
113	int batch_progress_limit;
114};
115
116struct idxd_pmu {
117	struct idxd_device *idxd;
118
119	struct perf_event *event_list[IDXD_PMU_EVENT_MAX];
120	int n_events;
121
122	DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX);
123
124	struct pmu pmu;
125	char name[IDXD_NAME_SIZE];
126
127	int n_counters;
128	int counter_width;
129	int n_event_categories;
130
131	bool per_counter_caps_supported;
132	unsigned long supported_event_categories;
133
134	unsigned long supported_filters;
135	int n_filters;
136};
137
138#define IDXD_MAX_PRIORITY	0xf
139
140enum {
141	COUNTER_FAULTS = 0,
142	COUNTER_FAULT_FAILS,
143	COUNTER_MAX
144};
145
146enum idxd_wq_state {
147	IDXD_WQ_DISABLED = 0,
148	IDXD_WQ_ENABLED,
149};
150
151enum idxd_wq_flag {
152	WQ_FLAG_DEDICATED = 0,
153	WQ_FLAG_BLOCK_ON_FAULT,
154	WQ_FLAG_ATS_DISABLE,
155	WQ_FLAG_PRS_DISABLE,
156};
157
158enum idxd_wq_type {
159	IDXD_WQT_NONE = 0,
160	IDXD_WQT_KERNEL,
161	IDXD_WQT_USER,
162};
163
164struct idxd_cdev {
165	struct idxd_wq *wq;
166	struct cdev cdev;
167	struct idxd_dev idxd_dev;
168	int minor;
169};
170
171#define DRIVER_NAME_SIZE		128
172
173#define WQ_NAME_SIZE   1024
174#define WQ_TYPE_SIZE   10
175
176#define WQ_DEFAULT_QUEUE_DEPTH		16
177#define WQ_DEFAULT_MAX_XFER		SZ_2M
178#define WQ_DEFAULT_MAX_BATCH		32
179
180enum idxd_op_type {
181	IDXD_OP_BLOCK = 0,
182	IDXD_OP_NONBLOCK = 1,
183};
184
185struct idxd_dma_chan {
186	struct dma_chan chan;
187	struct idxd_wq *wq;
188};
189
190struct idxd_wq {
191	void __iomem *portal;
192	u32 portal_offset;
193	unsigned int enqcmds_retries;
194	struct percpu_ref wq_active;
195	struct completion wq_dead;
196	struct completion wq_resurrect;
197	struct idxd_dev idxd_dev;
198	struct idxd_cdev *idxd_cdev;
199	struct wait_queue_head err_queue;
200	struct workqueue_struct *wq;
201	struct idxd_device *idxd;
202	int id;
203	struct idxd_irq_entry ie;
204	enum idxd_wq_type type;
205	struct idxd_group *group;
206	int client_count;
207	struct mutex wq_lock;	/* mutex for workqueue */
208	u32 size;
209	u32 threshold;
210	u32 priority;
211	enum idxd_wq_state state;
212	unsigned long flags;
213	union wqcfg *wqcfg;
214	unsigned long *opcap_bmap;
215
216	struct dsa_hw_desc **hw_descs;
217	int num_descs;
218	union {
219		struct dsa_completion_record *compls;
220		struct iax_completion_record *iax_compls;
221	};
222	dma_addr_t compls_addr;
223	int compls_size;
224	struct idxd_desc **descs;
225	struct sbitmap_queue sbq;
226	struct idxd_dma_chan *idxd_chan;
227	char name[WQ_NAME_SIZE + 1];
228	u64 max_xfer_bytes;
229	u32 max_batch_size;
230
231	/* Lock to protect upasid_xa access. */
232	struct mutex uc_lock;
233	struct xarray upasid_xa;
234
235	char driver_name[DRIVER_NAME_SIZE + 1];
236};
237
238struct idxd_engine {
239	struct idxd_dev idxd_dev;
240	int id;
241	struct idxd_group *group;
242	struct idxd_device *idxd;
243};
244
245/* shadow registers */
246struct idxd_hw {
247	u32 version;
248	union gen_cap_reg gen_cap;
249	union wq_cap_reg wq_cap;
250	union group_cap_reg group_cap;
251	union engine_cap_reg engine_cap;
252	struct opcap opcap;
253	u32 cmd_cap;
254	union iaa_cap_reg iaa_cap;
255};
256
257enum idxd_device_state {
258	IDXD_DEV_HALTED = -1,
259	IDXD_DEV_DISABLED = 0,
260	IDXD_DEV_ENABLED,
261};
262
263enum idxd_device_flag {
264	IDXD_FLAG_CONFIGURABLE = 0,
265	IDXD_FLAG_CMD_RUNNING,
266	IDXD_FLAG_PASID_ENABLED,
267	IDXD_FLAG_USER_PASID_ENABLED,
268};
269
270struct idxd_dma_dev {
271	struct idxd_device *idxd;
272	struct dma_device dma;
273};
274
275typedef int (*load_device_defaults_fn_t) (struct idxd_device *idxd);
276
277struct idxd_driver_data {
278	const char *name_prefix;
279	enum idxd_type type;
280	const struct device_type *dev_type;
281	int compl_size;
282	int align;
283	int evl_cr_off;
284	int cr_status_off;
285	int cr_result_off;
286	bool user_submission_safe;
287	load_device_defaults_fn_t load_device_defaults;
288};
289
290struct idxd_evl {
291	/* Lock to protect event log access. */
292	struct mutex lock;
293	void *log;
294	dma_addr_t dma;
295	/* Total size of event log = number of entries * entry size. */
296	unsigned int log_size;
297	/* The number of entries in the event log. */
298	u16 size;
299	unsigned long *bmap;
300	bool batch_fail[IDXD_MAX_BATCH_IDENT];
301};
302
303struct idxd_evl_fault {
304	struct work_struct work;
305	struct idxd_wq *wq;
306	u8 status;
307
308	/* make this last member always */
309	struct __evl_entry entry[];
310};
311
312struct idxd_device {
313	struct idxd_dev idxd_dev;
314	struct idxd_driver_data *data;
315	struct list_head list;
316	struct idxd_hw hw;
317	enum idxd_device_state state;
318	unsigned long flags;
319	int id;
320	int major;
321	u32 cmd_status;
322	struct idxd_irq_entry ie;	/* misc irq, msix 0 */
323
324	struct pci_dev *pdev;
325	void __iomem *reg_base;
326
327	spinlock_t dev_lock;	/* spinlock for device */
328	spinlock_t cmd_lock;	/* spinlock for device commands */
329	struct completion *cmd_done;
330	struct idxd_group **groups;
331	struct idxd_wq **wqs;
332	struct idxd_engine **engines;
333
334	struct iommu_sva *sva;
335	unsigned int pasid;
336
337	int num_groups;
338	int irq_cnt;
339	bool request_int_handles;
340
341	u32 msix_perm_offset;
342	u32 wqcfg_offset;
343	u32 grpcfg_offset;
344	u32 perfmon_offset;
345
346	u64 max_xfer_bytes;
347	u32 max_batch_size;
348	int max_groups;
349	int max_engines;
350	int max_rdbufs;
351	int max_wqs;
352	int max_wq_size;
353	int rdbuf_limit;
354	int nr_rdbufs;		/* non-reserved read buffers */
355	unsigned int wqcfg_size;
356	unsigned long *wq_enable_map;
357
358	union sw_err_reg sw_err;
359	wait_queue_head_t cmd_waitq;
360
361	struct idxd_dma_dev *idxd_dma;
362	struct workqueue_struct *wq;
363	struct work_struct work;
364
365	struct idxd_pmu *idxd_pmu;
366
367	unsigned long *opcap_bmap;
368	struct idxd_evl *evl;
369	struct kmem_cache *evl_cache;
370
371	struct dentry *dbgfs_dir;
372	struct dentry *dbgfs_evl_file;
373
374	bool user_submission_safe;
375
376	struct idxd_saved_states *idxd_saved;
377};
378
379struct idxd_saved_states {
380	struct idxd_device saved_idxd;
381	struct idxd_evl saved_evl;
382	struct idxd_engine **saved_engines;
383	struct idxd_wq **saved_wqs;
384	struct idxd_group **saved_groups;
385	unsigned long *saved_wq_enable_map;
386};
387
388static inline unsigned int evl_ent_size(struct idxd_device *idxd)
389{
390	return idxd->hw.gen_cap.evl_support ?
391	       (32 * (1 << idxd->hw.gen_cap.evl_support)) : 0;
392}
393
394static inline unsigned int evl_size(struct idxd_device *idxd)
395{
396	return idxd->evl->size * evl_ent_size(idxd);
397}
398
399struct crypto_ctx {
400	struct acomp_req *req;
401	struct crypto_tfm *tfm;
402	dma_addr_t src_addr;
403	dma_addr_t dst_addr;
404	bool compress;
405};
406
407/* IDXD software descriptor */
408struct idxd_desc {
409	union {
410		struct dsa_hw_desc *hw;
411		struct iax_hw_desc *iax_hw;
412	};
413	dma_addr_t desc_dma;
414	union {
415		struct dsa_completion_record *completion;
416		struct iax_completion_record *iax_completion;
417	};
418	dma_addr_t compl_dma;
419	union {
420		struct dma_async_tx_descriptor txd;
421		struct crypto_ctx crypto;
422	};
423	struct llist_node llnode;
424	struct list_head list;
425	int id;
426	int cpu;
427	struct idxd_wq *wq;
428};
429
430/*
431 * This is software defined error for the completion status. We overload the error code
432 * that will never appear in completion status and only SWERR register.
433 */
434enum idxd_completion_status {
435	IDXD_COMP_DESC_ABORT = 0xff,
436};
437
438#define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev
439#define wq_confdev(wq) &wq->idxd_dev.conf_dev
440#define engine_confdev(engine) &engine->idxd_dev.conf_dev
441#define group_confdev(group) &group->idxd_dev.conf_dev
442#define cdev_dev(cdev) &cdev->idxd_dev.conf_dev
443#define user_ctx_dev(ctx) (&(ctx)->idxd_dev.conf_dev)
444
445#define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev)
446#define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev)
447#define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev)
448
449static inline struct idxd_device_driver *wq_to_idxd_drv(struct idxd_wq *wq)
450{
451	struct device *dev = wq_confdev(wq);
452	struct idxd_device_driver *idxd_drv =
453		container_of(dev->driver, struct idxd_device_driver, drv);
454
455	return idxd_drv;
456}
457
458static inline struct idxd_device *confdev_to_idxd(struct device *dev)
459{
460	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
461
462	return idxd_dev_to_idxd(idxd_dev);
463}
464
465static inline struct idxd_wq *confdev_to_wq(struct device *dev)
466{
467	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
468
469	return idxd_dev_to_wq(idxd_dev);
470}
471
472static inline struct idxd_engine *confdev_to_engine(struct device *dev)
473{
474	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
475
476	return container_of(idxd_dev, struct idxd_engine, idxd_dev);
477}
478
479static inline struct idxd_group *confdev_to_group(struct device *dev)
480{
481	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
482
483	return container_of(idxd_dev, struct idxd_group, idxd_dev);
484}
485
486static inline struct idxd_cdev *dev_to_cdev(struct device *dev)
487{
488	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
489
490	return container_of(idxd_dev, struct idxd_cdev, idxd_dev);
491}
492
493static inline void idxd_dev_set_type(struct idxd_dev *idev, int type)
494{
495	if (type >= IDXD_DEV_MAX_TYPE) {
496		idev->type = IDXD_DEV_NONE;
497		return;
498	}
499
500	idev->type = type;
501}
502
503static inline struct idxd_irq_entry *idxd_get_ie(struct idxd_device *idxd, int idx)
504{
505	return (idx == 0) ? &idxd->ie : &idxd->wqs[idx - 1]->ie;
506}
507
508static inline struct idxd_wq *ie_to_wq(struct idxd_irq_entry *ie)
509{
510	return container_of(ie, struct idxd_wq, ie);
511}
512
513static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie)
514{
515	return container_of(ie, struct idxd_device, ie);
516}
517
518static inline void idxd_set_user_intr(struct idxd_device *idxd, bool enable)
519{
520	union gencfg_reg reg;
521
522	reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
523	reg.user_int_en = enable;
524	iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET);
525}
526
527extern const struct bus_type dsa_bus_type;
528
529extern bool support_enqcmd;
530extern struct ida idxd_ida;
531extern const struct device_type dsa_device_type;
532extern const struct device_type iax_device_type;
533extern const struct device_type idxd_wq_device_type;
534extern const struct device_type idxd_engine_device_type;
535extern const struct device_type idxd_group_device_type;
536
537static inline bool is_dsa_dev(struct idxd_dev *idxd_dev)
538{
539	return idxd_dev->type == IDXD_DEV_DSA;
540}
541
542static inline bool is_iax_dev(struct idxd_dev *idxd_dev)
543{
544	return idxd_dev->type == IDXD_DEV_IAX;
545}
546
547static inline bool is_idxd_dev(struct idxd_dev *idxd_dev)
548{
549	return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev);
550}
551
552static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev)
553{
554	return idxd_dev->type == IDXD_DEV_WQ;
555}
556
557static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq)
558{
559	if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0)
560		return true;
561	return false;
562}
563
564static inline bool is_idxd_wq_user(struct idxd_wq *wq)
565{
566	return wq->type == IDXD_WQT_USER;
567}
568
569static inline bool is_idxd_wq_kernel(struct idxd_wq *wq)
570{
571	return wq->type == IDXD_WQT_KERNEL;
572}
573
574static inline bool wq_dedicated(struct idxd_wq *wq)
575{
576	return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
577}
578
579static inline bool wq_shared(struct idxd_wq *wq)
580{
581	return !test_bit(WQ_FLAG_DEDICATED, &wq->flags);
582}
583
584static inline bool device_pasid_enabled(struct idxd_device *idxd)
585{
586	return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
587}
588
589static inline bool device_user_pasid_enabled(struct idxd_device *idxd)
590{
591	return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags);
592}
593
594static inline bool wq_pasid_enabled(struct idxd_wq *wq)
595{
596	return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) ||
597	       (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd));
598}
599
600static inline bool wq_shared_supported(struct idxd_wq *wq)
601{
602	return (support_enqcmd && wq_pasid_enabled(wq));
603}
604
605enum idxd_portal_prot {
606	IDXD_PORTAL_UNLIMITED = 0,
607	IDXD_PORTAL_LIMITED,
608};
609
610enum idxd_interrupt_type {
611	IDXD_IRQ_MSIX = 0,
612	IDXD_IRQ_IMS,
613};
614
615static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
616{
617	return prot * 0x1000;
618}
619
620static inline int idxd_get_wq_portal_full_offset(int wq_id,
621						 enum idxd_portal_prot prot)
622{
623	return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
624}
625
626#define IDXD_PORTAL_MASK	(PAGE_SIZE - 1)
627
628/*
629 * Even though this function can be accessed by multiple threads, it is safe to use.
630 * At worst the address gets used more than once before it gets incremented. We don't
631 * hit a threshold until iops becomes many million times a second. So the occasional
632 * reuse of the same address is tolerable compare to using an atomic variable. This is
633 * safe on a system that has atomic load/store for 32bit integers. Given that this is an
634 * Intel iEP device, that should not be a problem.
635 */
636static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq)
637{
638	int ofs = wq->portal_offset;
639
640	wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK;
641	return wq->portal + ofs;
642}
643
644static inline void idxd_wq_get(struct idxd_wq *wq)
645{
646	wq->client_count++;
647}
648
649static inline void idxd_wq_put(struct idxd_wq *wq)
650{
651	wq->client_count--;
652}
653
654static inline int idxd_wq_refcount(struct idxd_wq *wq)
655{
656	return wq->client_count;
657};
658
659static inline void idxd_wq_set_private(struct idxd_wq *wq, void *private)
660{
661	dev_set_drvdata(wq_confdev(wq), private);
662}
663
664static inline void *idxd_wq_get_private(struct idxd_wq *wq)
665{
666	return dev_get_drvdata(wq_confdev(wq));
667}
668
669/*
670 * Intel IAA does not support batch processing.
671 * The max batch size of device, max batch size of wq and
672 * max batch shift of wqcfg should be always 0 on IAA.
673 */
674static inline void idxd_set_max_batch_size(int idxd_type, struct idxd_device *idxd,
675					   u32 max_batch_size)
676{
677	if (idxd_type == IDXD_TYPE_IAX)
678		idxd->max_batch_size = 0;
679	else
680		idxd->max_batch_size = max_batch_size;
681}
682
683static inline void idxd_wq_set_max_batch_size(int idxd_type, struct idxd_wq *wq,
684					      u32 max_batch_size)
685{
686	if (idxd_type == IDXD_TYPE_IAX)
687		wq->max_batch_size = 0;
688	else
689		wq->max_batch_size = max_batch_size;
690}
691
692static inline void idxd_wqcfg_set_max_batch_shift(int idxd_type, union wqcfg *wqcfg,
693						  u32 max_batch_shift)
694{
695	if (idxd_type == IDXD_TYPE_IAX)
696		wqcfg->max_batch_shift = 0;
697	else
698		wqcfg->max_batch_shift = max_batch_shift;
699}
700
701static inline int idxd_wq_driver_name_match(struct idxd_wq *wq, struct device *dev)
702{
703	return (strncmp(wq->driver_name, dev->driver->name, strlen(dev->driver->name)) == 0);
704}
705
706#define MODULE_ALIAS_IDXD_DEVICE(type) MODULE_ALIAS("idxd:t" __stringify(type) "*")
707#define IDXD_DEVICES_MODALIAS_FMT "idxd:t%d"
708
709int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv,
710					struct module *module, const char *mod_name);
711#define idxd_driver_register(driver) \
712	__idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME)
713
714void idxd_driver_unregister(struct idxd_device_driver *idxd_drv);
715
716#define module_idxd_driver(__idxd_driver) \
717	module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister)
718
719void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);
720void idxd_dma_complete_txd(struct idxd_desc *desc,
721			   enum idxd_complete_type comp_type,
722			   bool free_desc, void *ctx, u32 *status);
723
724static inline void idxd_desc_complete(struct idxd_desc *desc,
725				      enum idxd_complete_type comp_type,
726				      bool free_desc)
727{
728	struct idxd_device_driver *drv;
729	u32 status;
730
731	drv = wq_to_idxd_drv(desc->wq);
732	if (drv->desc_complete)
733		drv->desc_complete(desc, comp_type, free_desc,
734				   &desc->txd, &status);
735}
736
737int idxd_register_devices(struct idxd_device *idxd);
738void idxd_unregister_devices(struct idxd_device *idxd);
739void idxd_wqs_quiesce(struct idxd_device *idxd);
740bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc);
741void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count);
742int idxd_load_iaa_device_defaults(struct idxd_device *idxd);
743
744/* device interrupt control */
745irqreturn_t idxd_misc_thread(int vec, void *data);
746irqreturn_t idxd_wq_thread(int irq, void *data);
747void idxd_mask_error_interrupts(struct idxd_device *idxd);
748void idxd_unmask_error_interrupts(struct idxd_device *idxd);
749
750/* device control */
751int idxd_device_drv_probe(struct idxd_dev *idxd_dev);
752int idxd_pci_probe_alloc(struct idxd_device *idxd, struct pci_dev *pdev,
753			 const struct pci_device_id *id);
754void idxd_device_drv_remove(struct idxd_dev *idxd_dev);
755int idxd_drv_enable_wq(struct idxd_wq *wq);
756void idxd_drv_disable_wq(struct idxd_wq *wq);
757int idxd_device_init_reset(struct idxd_device *idxd);
758int idxd_device_enable(struct idxd_device *idxd);
759int idxd_device_disable(struct idxd_device *idxd);
760void idxd_device_reset(struct idxd_device *idxd);
761void idxd_device_clear_state(struct idxd_device *idxd);
762int idxd_device_config(struct idxd_device *idxd);
763void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid);
764int idxd_device_load_config(struct idxd_device *idxd);
765int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle,
766				   enum idxd_interrupt_type irq_type);
767int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
768				   enum idxd_interrupt_type irq_type);
769
770/* work queue control */
771void idxd_wqs_unmap_portal(struct idxd_device *idxd);
772int idxd_wq_alloc_resources(struct idxd_wq *wq);
773void idxd_wq_free_resources(struct idxd_wq *wq);
774int idxd_wq_enable(struct idxd_wq *wq);
775int idxd_wq_disable(struct idxd_wq *wq, bool reset_config);
776void idxd_wq_drain(struct idxd_wq *wq);
777void idxd_wq_reset(struct idxd_wq *wq);
778int idxd_wq_map_portal(struct idxd_wq *wq);
779void idxd_wq_unmap_portal(struct idxd_wq *wq);
780int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid);
781int idxd_wq_disable_pasid(struct idxd_wq *wq);
782void __idxd_wq_quiesce(struct idxd_wq *wq);
783void idxd_wq_quiesce(struct idxd_wq *wq);
784int idxd_wq_init_percpu_ref(struct idxd_wq *wq);
785void idxd_wq_free_irq(struct idxd_wq *wq);
786int idxd_wq_request_irq(struct idxd_wq *wq);
787
788/* submission */
789int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
790struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);
791int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc);
792
793/* dmaengine */
794int idxd_register_dma_device(struct idxd_device *idxd);
795void idxd_unregister_dma_device(struct idxd_device *idxd);
796
797/* cdev */
798int idxd_cdev_register(void);
799void idxd_cdev_remove(void);
800int idxd_cdev_get_major(struct idxd_device *idxd);
801int idxd_wq_add_cdev(struct idxd_wq *wq);
802void idxd_wq_del_cdev(struct idxd_wq *wq);
803int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr,
804		 void *buf, int len);
805void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index);
806
807/* perfmon */
808#if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON)
809int perfmon_pmu_init(struct idxd_device *idxd);
810void perfmon_pmu_remove(struct idxd_device *idxd);
811void perfmon_counter_overflow(struct idxd_device *idxd);
812#else
813static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; }
814static inline void perfmon_pmu_remove(struct idxd_device *idxd) {}
815static inline void perfmon_counter_overflow(struct idxd_device *idxd) {}
816#endif
817
818/* debugfs */
819int idxd_device_init_debugfs(struct idxd_device *idxd);
820void idxd_device_remove_debugfs(struct idxd_device *idxd);
821int idxd_init_debugfs(void);
822void idxd_remove_debugfs(void);
823
824#endif