priv.h

  1/* SPDX-License-Identifier: MIT */
  2#ifndef __NVKM_FIFO_PRIV_H__
  3#define __NVKM_FIFO_PRIV_H__
  4#define nvkm_fifo(p) container_of((p), struct nvkm_fifo, engine)
  5#include <engine/fifo.h>
  6#include <core/enum.h>
  7struct nvkm_cctx;
  8struct nvkm_cgrp;
  9struct nvkm_chan;
 10struct nvkm_engn;
 11struct nvkm_memory;
 12struct nvkm_runl;
 13struct nvkm_runq;
 14struct nvkm_vctx;
 15
 16struct nvkm_fifo_func {
 17	void (*dtor)(struct nvkm_fifo *);
 18
 19	int (*chid_nr)(struct nvkm_fifo *);
 20	int (*chid_ctor)(struct nvkm_fifo *, int nr);
 21	int (*runq_nr)(struct nvkm_fifo *);
 22	int (*runl_ctor)(struct nvkm_fifo *);
 23
 24	void (*init)(struct nvkm_fifo *);
 25	void (*init_pbdmas)(struct nvkm_fifo *, u32 mask);
 26
 27	irqreturn_t (*intr)(struct nvkm_inth *);
 28	void (*intr_mmu_fault_unit)(struct nvkm_fifo *, int unit);
 29	void (*intr_ctxsw_timeout)(struct nvkm_fifo *, u32 engm);
 30
 31	const struct nvkm_fifo_func_mmu_fault {
 32		void (*recover)(struct nvkm_fifo *, struct nvkm_fault_data *);
 33		const struct nvkm_enum *access;
 34		const struct nvkm_enum *engine;
 35		const struct nvkm_enum *reason;
 36		const struct nvkm_enum *hubclient;
 37		const struct nvkm_enum *gpcclient;
 38	} *mmu_fault;
 39
 40	void (*pause)(struct nvkm_fifo *, unsigned long *);
 41	void (*start)(struct nvkm_fifo *, unsigned long *);
 42
 43	int (*nonstall_ctor)(struct nvkm_fifo *);
 44	void (*nonstall_dtor)(struct nvkm_fifo *);
 45	const struct nvkm_event_func *nonstall;
 46
 47	const struct nvkm_runl_func *runl;
 48	const struct nvkm_runq_func *runq;
 49	const struct nvkm_engn_func *engn;
 50	const struct nvkm_engn_func *engn_sw;
 51	const struct nvkm_engn_func *engn_ce;
 52
 53	struct nvkm_fifo_func_cgrp {
 54		struct nvkm_sclass user;
 55		const struct nvkm_cgrp_func *func;
 56		bool force;
 57	} cgrp;
 58
 59	struct nvkm_fifo_func_chan {
 60		struct nvkm_sclass user;
 61		const struct nvkm_chan_func *func;
 62	} chan;
 63};
 64
 65int r535_fifo_new(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int,
 66		  struct nvkm_fifo **);
 67int nvkm_fifo_new_(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int,
 68		   struct nvkm_fifo **);
 69
 70int nv04_fifo_chid_ctor(struct nvkm_fifo *, int);
 71int nv04_fifo_runl_ctor(struct nvkm_fifo *);
 72void nv04_fifo_init(struct nvkm_fifo *);
 73irqreturn_t nv04_fifo_intr(struct nvkm_inth *);
 74void nv04_fifo_pause(struct nvkm_fifo *, unsigned long *);
 75void nv04_fifo_start(struct nvkm_fifo *, unsigned long *);
 76extern const struct nvkm_runl_func nv04_runl;
 77extern const struct nvkm_engn_func nv04_engn;
 78extern const struct nvkm_cgrp_func nv04_cgrp;
 79extern const struct nvkm_chan_func_inst nv04_chan_inst;
 80extern const struct nvkm_chan_func_userd nv04_chan_userd;
 81void nv04_chan_ramfc_clear(struct nvkm_chan *);
 82void nv04_chan_start(struct nvkm_chan *);
 83void nv04_chan_stop(struct nvkm_chan *);
 84void nv04_eobj_ramht_del(struct nvkm_chan *, int);
 85
 86int nv10_fifo_chid_nr(struct nvkm_fifo *);
 87
 88int nv50_fifo_chid_nr(struct nvkm_fifo *);
 89int nv50_fifo_chid_ctor(struct nvkm_fifo *, int);
 90void nv50_fifo_init(struct nvkm_fifo *);
 91extern const struct nvkm_runl_func nv50_runl;
 92int nv50_runl_update(struct nvkm_runl *);
 93int nv50_runl_wait(struct nvkm_runl *);
 94extern const struct nvkm_engn_func nv50_engn_sw;
 95extern const struct nvkm_chan_func_inst nv50_chan_inst;
 96extern const struct nvkm_chan_func_userd nv50_chan_userd;
 97void nv50_chan_unbind(struct nvkm_chan *);
 98void nv50_chan_start(struct nvkm_chan *);
 99void nv50_chan_stop(struct nvkm_chan *);
100void nv50_chan_preempt(struct nvkm_chan *);
101int nv50_eobj_ramht_add(struct nvkm_engn *, struct nvkm_object *, struct nvkm_chan *);
102void nv50_eobj_ramht_del(struct nvkm_chan *, int);
103
104extern const struct nvkm_event_func g84_fifo_nonstall;
105extern const struct nvkm_engn_func g84_engn;
106extern const struct nvkm_chan_func g84_chan;
107
108int gf100_fifo_chid_ctor(struct nvkm_fifo *, int);
109int gf100_fifo_runq_nr(struct nvkm_fifo *);
110bool gf100_fifo_intr_pbdma(struct nvkm_fifo *);
111void gf100_fifo_intr_mmu_fault(struct nvkm_fifo *);
112void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
113void gf100_fifo_intr_sched(struct nvkm_fifo *);
114void gf100_fifo_intr_ctxsw_timeout(struct nvkm_fifo *, u32);
115void gf100_fifo_mmu_fault_recover(struct nvkm_fifo *, struct nvkm_fault_data *);
116extern const struct nvkm_enum gf100_fifo_mmu_fault_access[];
117extern const struct nvkm_event_func gf100_fifo_nonstall;
118bool gf100_runl_preempt_pending(struct nvkm_runl *);
119void gf100_runq_init(struct nvkm_runq *);
120bool gf100_runq_intr(struct nvkm_runq *, struct nvkm_runl *);
121void gf100_engn_mmu_fault_trigger(struct nvkm_engn *);
122bool gf100_engn_mmu_fault_triggered(struct nvkm_engn *);
123extern const struct nvkm_engn_func gf100_engn_sw;
124extern const struct nvkm_chan_func_inst gf100_chan_inst;
125void gf100_chan_userd_clear(struct nvkm_chan *);
126void gf100_chan_preempt(struct nvkm_chan *);
127
128int gk104_fifo_chid_nr(struct nvkm_fifo *);
129int gk104_fifo_runl_ctor(struct nvkm_fifo *);
130void gk104_fifo_init(struct nvkm_fifo *);
131void gk104_fifo_init_pbdmas(struct nvkm_fifo *, u32);
132irqreturn_t gk104_fifo_intr(struct nvkm_inth *);
133void gk104_fifo_intr_runlist(struct nvkm_fifo *);
134void gk104_fifo_intr_chsw(struct nvkm_fifo *);
135void gk104_fifo_intr_bind(struct nvkm_fifo *);
136extern const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault;
137extern const struct nvkm_enum gk104_fifo_mmu_fault_reason[];
138extern const struct nvkm_enum gk104_fifo_mmu_fault_hubclient[];
139extern const struct nvkm_enum gk104_fifo_mmu_fault_gpcclient[];
140void gk104_runl_insert_chan(struct nvkm_chan *, struct nvkm_memory *, u64);
141void gk104_runl_commit(struct nvkm_runl *, struct nvkm_memory *, u32, int);
142bool gk104_runl_pending(struct nvkm_runl *);
143void gk104_runl_block(struct nvkm_runl *, u32);
144void gk104_runl_allow(struct nvkm_runl *, u32);
145void gk104_runl_fault_clear(struct nvkm_runl *);
146extern const struct nvkm_runq_func gk104_runq;
147void gk104_runq_init(struct nvkm_runq *);
148bool gk104_runq_intr(struct nvkm_runq *, struct nvkm_runl *);
149extern const struct nvkm_bitfield gk104_runq_intr_0_names[];
150bool gk104_runq_idle(struct nvkm_runq *);
151extern const struct nvkm_engn_func gk104_engn;
152bool gk104_engn_chsw(struct nvkm_engn *);
153int gk104_engn_cxid(struct nvkm_engn *, bool *cgid);
154int gk104_ectx_ctor(struct nvkm_engn *, struct nvkm_vctx *);
155extern const struct nvkm_engn_func gk104_engn_ce;
156extern const struct nvkm_chan_func_userd gk104_chan_userd;
157extern const struct nvkm_chan_func_ramfc gk104_chan_ramfc;
158void gk104_chan_bind(struct nvkm_chan *);
159void gk104_chan_bind_inst(struct nvkm_chan *);
160void gk104_chan_unbind(struct nvkm_chan *);
161void gk104_chan_start(struct nvkm_chan *);
162void gk104_chan_stop(struct nvkm_chan *);
163
164int gk110_fifo_chid_ctor(struct nvkm_fifo *, int);
165extern const struct nvkm_runl_func gk110_runl;
166extern const struct nvkm_cgrp_func gk110_cgrp;
167void gk110_runl_insert_cgrp(struct nvkm_cgrp *, struct nvkm_memory *, u64);
168extern const struct nvkm_chan_func gk110_chan;
169void gk110_chan_preempt(struct nvkm_chan *);
170
171extern const struct nvkm_runq_func gk208_runq;
172void gk208_runq_init(struct nvkm_runq *);
173
174void gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
175extern const struct nvkm_fifo_func_mmu_fault gm107_fifo_mmu_fault;
176extern const struct nvkm_runl_func gm107_runl;
177extern const struct nvkm_chan_func gm107_chan;
178
179int gm200_fifo_chid_nr(struct nvkm_fifo *);
180int gm200_fifo_runq_nr(struct nvkm_fifo *);
181
182extern const struct nvkm_enum gv100_fifo_mmu_fault_access[];
183extern const struct nvkm_enum gv100_fifo_mmu_fault_reason[];
184extern const struct nvkm_enum gv100_fifo_mmu_fault_hubclient[];
185extern const struct nvkm_enum gv100_fifo_mmu_fault_gpcclient[];
186void gv100_runl_insert_cgrp(struct nvkm_cgrp *, struct nvkm_memory *, u64);
187void gv100_runl_insert_chan(struct nvkm_chan *, struct nvkm_memory *, u64);
188void gv100_runl_preempt(struct nvkm_runl *);
189extern const struct nvkm_runq_func gv100_runq;
190extern const struct nvkm_engn_func gv100_engn;
191void gv100_ectx_bind(struct nvkm_engn *, struct nvkm_cctx *, struct nvkm_chan *);
192extern const struct nvkm_engn_func gv100_engn_ce;
193int gv100_ectx_ce_ctor(struct nvkm_engn *, struct nvkm_vctx *);
194void gv100_ectx_ce_bind(struct nvkm_engn *, struct nvkm_cctx *, struct nvkm_chan *);
195extern const struct nvkm_chan_func_userd gv100_chan_userd;
196extern const struct nvkm_chan_func_ramfc gv100_chan_ramfc;
197
198void tu102_fifo_intr_ctxsw_timeout_info(struct nvkm_engn *, u32 info);
199extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault;
200u32 tu102_chan_doorbell_handle(struct nvkm_chan *);
201
202int ga100_fifo_runl_ctor(struct nvkm_fifo *);
203int ga100_fifo_nonstall_ctor(struct nvkm_fifo *);
204void ga100_fifo_nonstall_dtor(struct nvkm_fifo *);
205extern const struct nvkm_event_func ga100_fifo_nonstall;
206extern const struct nvkm_runl_func ga100_runl;
207extern const struct nvkm_runq_func ga100_runq;
208extern const struct nvkm_engn_func ga100_engn;
209extern const struct nvkm_engn_func ga100_engn_ce;
210extern const struct nvkm_cgrp_func ga100_cgrp;
211extern const struct nvkm_chan_func ga100_chan;
212
213u32 gb202_chan_doorbell_handle(struct nvkm_chan *);
214
215int nvkm_uchan_new(struct nvkm_fifo *, struct nvkm_cgrp *, const struct nvkm_oclass *,
216		   void *argv, u32 argc, struct nvkm_object **);
217int nvkm_ucgrp_new(struct nvkm_fifo *, const struct nvkm_oclass *, void *argv, u32 argc,
218		   struct nvkm_object **);
219#endif