chan.c

  1/* SPDX-License-Identifier: MIT
  2 *
  3 * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
  4 */
  5#include <nvif/chan.h>
  6
  7static void
  8nvif_chan_gpfifo_push_kick(struct nvif_push *push)
  9{
 10	struct nvif_chan *chan = container_of(push, typeof(*chan), push);
 11	u32 put = push->bgn - (u32 *)chan->push.mem.object.map.ptr;
 12	u32 cnt;
 13
 14	if (chan->func->gpfifo.post) {
 15		if (push->end - push->cur < chan->func->gpfifo.post_size)
 16			push->end = push->cur + chan->func->gpfifo.post_size;
 17
 18		WARN_ON(nvif_chan_gpfifo_post(chan));
 19	}
 20
 21	cnt = push->cur - push->bgn;
 22
 23	chan->func->gpfifo.push(chan, true, chan->push.addr + (put << 2), cnt << 2, false);
 24	chan->func->gpfifo.kick(chan);
 25}
 26
 27static int
 28nvif_chan_gpfifo_push_wait(struct nvif_push *push, u32 push_nr)
 29{
 30	struct nvif_chan *chan = container_of(push, typeof(*chan), push);
 31
 32	return nvif_chan_gpfifo_wait(chan, 1, push_nr);
 33}
 34
 35int
 36nvif_chan_gpfifo_post(struct nvif_chan *chan)
 37{
 38	const u32 *map = chan->push.mem.object.map.ptr;
 39	const u32 pbptr = (chan->push.cur - map) + chan->func->gpfifo.post_size;
 40	const u32 gpptr = (chan->gpfifo.cur + 1) & chan->gpfifo.max;
 41
 42	if (!chan->func->gpfifo.post)
 43		return 0;
 44
 45	return chan->func->gpfifo.post(chan, gpptr, pbptr);
 46}
 47
 48void
 49nvif_chan_gpfifo_push(struct nvif_chan *chan, u64 addr, u32 size, bool no_prefetch)
 50{
 51	chan->func->gpfifo.push(chan, false, addr, size, no_prefetch);
 52}
 53
 54int
 55nvif_chan_gpfifo_wait(struct nvif_chan *chan, u32 gpfifo_nr, u32 push_nr)
 56{
 57	struct nvif_push *push = &chan->push;
 58	int ret = 0, time = 1000000;
 59
 60	if (gpfifo_nr) {
 61		/* Account for pushbuf space needed by nvif_chan_gpfifo_post(),
 62		 * if used after pushing userspace GPFIFO entries.
 63		 */
 64		if (chan->func->gpfifo.post)
 65			push_nr += chan->func->gpfifo.post_size;
 66	}
 67
 68	/* Account for the GPFIFO entry needed to submit pushbuf. */
 69	if (push_nr)
 70		gpfifo_nr++;
 71
 72	/* Wait for space in main push buffer. */
 73	if (push->cur + push_nr > push->end) {
 74		ret = nvif_chan_dma_wait(chan, push_nr);
 75		if (ret)
 76			return ret;
 77	}
 78
 79	/* Wait for GPFIFO space. */
 80	while (chan->gpfifo.free < gpfifo_nr) {
 81		chan->gpfifo.free = chan->func->gpfifo.read_get(chan) - chan->gpfifo.cur - 1;
 82		if (chan->gpfifo.free < 0)
 83			chan->gpfifo.free += chan->gpfifo.max + 1;
 84
 85		if (chan->gpfifo.free < gpfifo_nr) {
 86			if (!time--)
 87				return -ETIMEDOUT;
 88			udelay(1);
 89		}
 90	}
 91
 92	return 0;
 93}
 94
 95void
 96nvif_chan_gpfifo_ctor(const struct nvif_chan_func *func, void *userd, void *gpfifo, u32 gpfifo_size,
 97		      void *push, u64 push_addr, u32 push_size, struct nvif_chan *chan)
 98{
 99	chan->func = func;
100
101	chan->userd.map.ptr = userd;
102
103	chan->gpfifo.map.ptr = gpfifo;
104	chan->gpfifo.max = (gpfifo_size >> 3) - 1;
105	chan->gpfifo.free = chan->gpfifo.max;
106
107	chan->push.mem.object.map.ptr = push;
108	chan->push.wait = nvif_chan_gpfifo_push_wait;
109	chan->push.kick = nvif_chan_gpfifo_push_kick;
110	chan->push.addr = push_addr;
111	chan->push.hw.max = push_size >> 2;
112	chan->push.bgn = chan->push.cur = chan->push.end = push;
113}
114
115int
116nvif_chan_dma_wait(struct nvif_chan *chan, u32 nr)
117{
118	struct nvif_push *push = &chan->push;
119	u32 cur = push->cur - (u32 *)push->mem.object.map.ptr;
120	u32 free, time = 1000000;
121
122	nr += chan->func->gpfifo.post_size;
123
124	do {
125		u32 get = chan->func->push.read_get(chan);
126
127		if (get <= cur) {
128			free = push->hw.max - cur;
129			if (free >= nr)
130				break;
131
132			PUSH_KICK(push);
133
134			while (get == 0) {
135				get = chan->func->push.read_get(chan);
136				if (get == 0) {
137					if (!time--)
138						return -ETIMEDOUT;
139					udelay(1);
140				}
141			}
142
143			cur = 0;
144		}
145
146		free = get - cur - 1;
147
148		if (free < nr) {
149			if (!time--)
150				return -ETIMEDOUT;
151			udelay(1);
152		}
153	} while (free < nr);
154
155	push->bgn = (u32 *)push->mem.object.map.ptr + cur;
156	push->cur = push->bgn;
157	push->end = push->bgn + free - chan->func->gpfifo.post_size;
158	return 0;
159}