nv50.c

  1/*
  2 * Copyright 2012 Red Hat Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 * Authors: Ben Skeggs
 23 */
 24#define nv50_instmem(p) container_of((p), struct nv50_instmem, base)
 25#include "priv.h"
 26
 27#include <core/memory.h>
 28#include <subdev/bar.h>
 29#include <subdev/fb.h>
 30#include <subdev/gsp.h>
 31#include <subdev/mmu.h>
 32
 33struct nv50_instmem {
 34	struct nvkm_instmem base;
 35	u64 addr;
 36
 37	/* Mappings that can be evicted when BAR2 space has been exhausted. */
 38	struct list_head lru;
 39};
 40
 41/******************************************************************************
 42 * instmem object implementation
 43 *****************************************************************************/
 44#define nv50_instobj(p) container_of((p), struct nv50_instobj, base.memory)
 45
 46struct nv50_instobj {
 47	struct nvkm_instobj base;
 48	struct nv50_instmem *imem;
 49	struct nvkm_memory *ram;
 50	struct nvkm_vma *bar;
 51	refcount_t maps;
 52	void *map;
 53	struct list_head lru;
 54};
 55
 56static void
 57nv50_instobj_wr32_slow(struct nvkm_memory *memory, u64 offset, u32 data)
 58{
 59	struct nv50_instobj *iobj = nv50_instobj(memory);
 60	struct nv50_instmem *imem = iobj->imem;
 61	struct nvkm_device *device = imem->base.subdev.device;
 62	u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL;
 63	u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL;
 64	unsigned long flags;
 65
 66	spin_lock_irqsave(&imem->base.lock, flags);
 67	if (unlikely(imem->addr != base)) {
 68		imem->base.func->set_bar0_window_addr(device, base);
 69		imem->addr = base;
 70	}
 71	nvkm_wr32(device, 0x700000 + addr, data);
 72	spin_unlock_irqrestore(&imem->base.lock, flags);
 73}
 74
 75static u32
 76nv50_instobj_rd32_slow(struct nvkm_memory *memory, u64 offset)
 77{
 78	struct nv50_instobj *iobj = nv50_instobj(memory);
 79	struct nv50_instmem *imem = iobj->imem;
 80	struct nvkm_device *device = imem->base.subdev.device;
 81	u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL;
 82	u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL;
 83	u32 data;
 84	unsigned long flags;
 85
 86	spin_lock_irqsave(&imem->base.lock, flags);
 87	if (unlikely(imem->addr != base)) {
 88		imem->base.func->set_bar0_window_addr(device, base);
 89		imem->addr = base;
 90	}
 91	data = nvkm_rd32(device, 0x700000 + addr);
 92	spin_unlock_irqrestore(&imem->base.lock, flags);
 93	return data;
 94}
 95
 96static const struct nvkm_memory_ptrs
 97nv50_instobj_slow = {
 98	.rd32 = nv50_instobj_rd32_slow,
 99	.wr32 = nv50_instobj_wr32_slow,
100};
101
102static void
103nv50_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
104{
105	iowrite32_native(data, nv50_instobj(memory)->map + offset);
106}
107
108static u32
109nv50_instobj_rd32(struct nvkm_memory *memory, u64 offset)
110{
111	return ioread32_native(nv50_instobj(memory)->map + offset);
112}
113
114static const struct nvkm_memory_ptrs
115nv50_instobj_fast = {
116	.rd32 = nv50_instobj_rd32,
117	.wr32 = nv50_instobj_wr32,
118};
119
120static void
121nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm)
122{
123	struct nv50_instmem *imem = iobj->imem;
124	struct nv50_instobj *eobj;
125	struct nvkm_memory *memory = &iobj->base.memory;
126	struct nvkm_subdev *subdev = &imem->base.subdev;
127	struct nvkm_device *device = subdev->device;
128	struct nvkm_vma *bar = NULL, *ebar;
129	u64 size = nvkm_memory_size(memory);
130	void *emap;
131	int ret;
132
133	/* Attempt to allocate BAR2 address-space and map the object
134	 * into it.  The lock has to be dropped while doing this due
135	 * to the possibility of recursion for page table allocation.
136	 */
137	mutex_unlock(&imem->base.mutex);
138	while ((ret = nvkm_vmm_get(vmm, 12, size, &bar))) {
139		/* Evict unused mappings, and keep retrying until we either
140		 * succeed,or there's no more objects left on the LRU.
141		 */
142		mutex_lock(&imem->base.mutex);
143		eobj = list_first_entry_or_null(&imem->lru, typeof(*eobj), lru);
144		if (eobj) {
145			nvkm_debug(subdev, "evict %016llx %016llx @ %016llx\n",
146				   nvkm_memory_addr(&eobj->base.memory),
147				   nvkm_memory_size(&eobj->base.memory),
148				   eobj->bar->addr);
149			list_del_init(&eobj->lru);
150			ebar = eobj->bar;
151			eobj->bar = NULL;
152			emap = eobj->map;
153			eobj->map = NULL;
154		}
155		mutex_unlock(&imem->base.mutex);
156		if (!eobj)
157			break;
158		iounmap(emap);
159		nvkm_vmm_put(vmm, &ebar);
160	}
161
162	if (ret == 0)
163		ret = nvkm_memory_map(memory, 0, vmm, bar, NULL, 0);
164	mutex_lock(&imem->base.mutex);
165	if (ret || iobj->bar) {
166		/* We either failed, or another thread beat us. */
167		mutex_unlock(&imem->base.mutex);
168		nvkm_vmm_put(vmm, &bar);
169		mutex_lock(&imem->base.mutex);
170		return;
171	}
172
173	/* Make the mapping visible to the host. */
174	iobj->bar = bar;
175	iobj->map = ioremap_wc(device->func->resource_addr(device, NVKM_BAR2_INST) +
176			       (u32)iobj->bar->addr, size);
177	if (!iobj->map) {
178		nvkm_warn(subdev, "PRAMIN ioremap failed\n");
179		nvkm_vmm_put(vmm, &iobj->bar);
180	}
181}
182
183static int
184nv50_instobj_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm,
185		 struct nvkm_vma *vma, void *argv, u32 argc)
186{
187	memory = nv50_instobj(memory)->ram;
188	return nvkm_memory_map(memory, offset, vmm, vma, argv, argc);
189}
190
191static void
192nv50_instobj_release(struct nvkm_memory *memory)
193{
194	struct nv50_instobj *iobj = nv50_instobj(memory);
195	struct nv50_instmem *imem = iobj->imem;
196	struct nvkm_subdev *subdev = &imem->base.subdev;
197
198	wmb();
199	nvkm_bar_flush(subdev->device->bar);
200
201	if (refcount_dec_and_mutex_lock(&iobj->maps, &imem->base.mutex)) {
202		/* Add the now-unused mapping to the LRU instead of directly
203		 * unmapping it here, in case we need to map it again later.
204		 */
205		if (likely(iobj->lru.next) && iobj->map) {
206			BUG_ON(!list_empty(&iobj->lru));
207			list_add_tail(&iobj->lru, &imem->lru);
208		}
209
210		/* Switch back to NULL accessors when last map is gone. */
211		iobj->base.memory.ptrs = NULL;
212		mutex_unlock(&imem->base.mutex);
213	}
214}
215
216static void __iomem *
217nv50_instobj_acquire(struct nvkm_memory *memory)
218{
219	struct nv50_instobj *iobj = nv50_instobj(memory);
220	struct nvkm_instmem *imem = &iobj->imem->base;
221	struct nvkm_vmm *vmm;
222	void __iomem *map = NULL;
223
224	/* Already mapped? */
225	if (refcount_inc_not_zero(&iobj->maps)) {
226		/* read barrier match the wmb on refcount set */
227		smp_rmb();
228		return iobj->map;
229	}
230
231	/* Take the lock, and re-check that another thread hasn't
232	 * already mapped the object in the meantime.
233	 */
234	mutex_lock(&imem->mutex);
235	if (refcount_inc_not_zero(&iobj->maps)) {
236		mutex_unlock(&imem->mutex);
237		return iobj->map;
238	}
239
240	/* Attempt to get a direct CPU mapping of the object. */
241	if ((vmm = nvkm_bar_bar2_vmm(imem->subdev.device))) {
242		if (!iobj->map)
243			nv50_instobj_kmap(iobj, vmm);
244		map = iobj->map;
245	}
246
247	if (!refcount_inc_not_zero(&iobj->maps)) {
248		/* Exclude object from eviction while it's being accessed. */
249		if (likely(iobj->lru.next))
250			list_del_init(&iobj->lru);
251
252		if (map)
253			iobj->base.memory.ptrs = &nv50_instobj_fast;
254		else
255			iobj->base.memory.ptrs = &nv50_instobj_slow;
256		/* barrier to ensure the ptrs are written before refcount is set */
257		smp_wmb();
258		refcount_set(&iobj->maps, 1);
259	}
260
261	mutex_unlock(&imem->mutex);
262	return map;
263}
264
265static void
266nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vmm *vmm)
267{
268	struct nv50_instobj *iobj = nv50_instobj(memory);
269	struct nvkm_instmem *imem = &iobj->imem->base;
270
271	/* Exclude bootstrapped objects (ie. the page tables for the
272	 * instmem BAR itself) from eviction.
273	 */
274	mutex_lock(&imem->mutex);
275	if (likely(iobj->lru.next)) {
276		list_del_init(&iobj->lru);
277		iobj->lru.next = NULL;
278	}
279
280	nv50_instobj_kmap(iobj, vmm);
281	nvkm_instmem_boot(imem);
282	mutex_unlock(&imem->mutex);
283}
284
285static u64
286nv50_instobj_size(struct nvkm_memory *memory)
287{
288	return nvkm_memory_size(nv50_instobj(memory)->ram);
289}
290
291static u64
292nv50_instobj_addr(struct nvkm_memory *memory)
293{
294	return nvkm_memory_addr(nv50_instobj(memory)->ram);
295}
296
297static u64
298nv50_instobj_bar2(struct nvkm_memory *memory)
299{
300	struct nv50_instobj *iobj = nv50_instobj(memory);
301	u64 addr = ~0ULL;
302	if (nv50_instobj_acquire(&iobj->base.memory)) {
303		iobj->lru.next = NULL; /* Exclude from eviction. */
304		addr = iobj->bar->addr;
305	}
306	nv50_instobj_release(&iobj->base.memory);
307	return addr;
308}
309
310static enum nvkm_memory_target
311nv50_instobj_target(struct nvkm_memory *memory)
312{
313	return nvkm_memory_target(nv50_instobj(memory)->ram);
314}
315
316static void *
317nv50_instobj_dtor(struct nvkm_memory *memory)
318{
319	struct nv50_instobj *iobj = nv50_instobj(memory);
320	struct nvkm_instmem *imem = &iobj->imem->base;
321	struct nvkm_vma *bar;
322	void *map;
323
324	mutex_lock(&imem->mutex);
325	if (likely(iobj->lru.next))
326		list_del(&iobj->lru);
327	map = iobj->map;
328	bar = iobj->bar;
329	mutex_unlock(&imem->mutex);
330
331	if (map) {
332		struct nvkm_vmm *vmm = nvkm_bar_bar2_vmm(imem->subdev.device);
333		iounmap(map);
334		if (likely(vmm)) /* Can be NULL during BAR destructor. */
335			nvkm_vmm_put(vmm, &bar);
336	}
337
338	nvkm_memory_unref(&iobj->ram);
339	nvkm_instobj_dtor(imem, &iobj->base);
340	return iobj;
341}
342
343static const struct nvkm_memory_func
344nv50_instobj_func = {
345	.dtor = nv50_instobj_dtor,
346	.target = nv50_instobj_target,
347	.bar2 = nv50_instobj_bar2,
348	.addr = nv50_instobj_addr,
349	.size = nv50_instobj_size,
350	.boot = nv50_instobj_boot,
351	.acquire = nv50_instobj_acquire,
352	.release = nv50_instobj_release,
353	.map = nv50_instobj_map,
354};
355
356int
357nv50_instobj_wrap(struct nvkm_instmem *base,
358		  struct nvkm_memory *memory, struct nvkm_memory **pmemory)
359{
360	struct nv50_instmem *imem = nv50_instmem(base);
361	struct nv50_instobj *iobj;
362
363	if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL)))
364		return -ENOMEM;
365	*pmemory = &iobj->base.memory;
366
367	nvkm_instobj_ctor(&nv50_instobj_func, &imem->base, &iobj->base);
368	iobj->imem = imem;
369	refcount_set(&iobj->maps, 0);
370	INIT_LIST_HEAD(&iobj->lru);
371
372	iobj->ram = nvkm_memory_ref(memory);
373	return 0;
374}
375
376int
377nv50_instobj_new(struct nvkm_instmem *imem, u32 size, u32 align, bool zero,
378		 struct nvkm_memory **pmemory)
379{
380	u8 page = max(order_base_2(align), 12);
381	struct nvkm_memory *ram;
382	int ret;
383
384	ret = nvkm_ram_get(imem->subdev.device, 0, 1, page, size, true, true, &ram);
385	if (ret)
386		return ret;
387
388	ret = nv50_instobj_wrap(imem, ram, pmemory);
389	nvkm_memory_unref(&ram);
390	return ret;
391}
392
393/******************************************************************************
394 * instmem subdev implementation
395 *****************************************************************************/
396
397static void
398nv50_instmem_set_bar0_window_addr(struct nvkm_device *device, u64 addr)
399{
400	nvkm_wr32(device, 0x001700, addr >> 16);
401}
402
403void
404nv50_instmem_fini(struct nvkm_instmem *base)
405{
406	nv50_instmem(base)->addr = ~0ULL;
407}
408
409static void *
410nv50_instmem_dtor(struct nvkm_instmem *base)
411{
412	return nv50_instmem(base);
413}
414
415static const struct nvkm_instmem_func
416nv50_instmem = {
417	.dtor = nv50_instmem_dtor,
418	.fini = nv50_instmem_fini,
419	.suspend = nv04_instmem_suspend,
420	.resume = nv04_instmem_resume,
421	.memory_new = nv50_instobj_new,
422	.memory_wrap = nv50_instobj_wrap,
423	.zero = false,
424	.set_bar0_window_addr = nv50_instmem_set_bar0_window_addr,
425};
426
427int
428nv50_instmem_new_(const struct nvkm_instmem_func *func,
429		  struct nvkm_device *device, enum nvkm_subdev_type type, int inst,
430		  struct nvkm_instmem **pimem)
431{
432	struct nv50_instmem *imem;
433
434	if (!(imem = kzalloc(sizeof(*imem), GFP_KERNEL)))
435		return -ENOMEM;
436	nvkm_instmem_ctor(func, device, type, inst, &imem->base);
437	INIT_LIST_HEAD(&imem->lru);
438	*pimem = &imem->base;
439	return 0;
440}
441
442int
443nv50_instmem_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst,
444		 struct nvkm_instmem **pimem)
445{
446	if (nvkm_gsp_rm(device->gsp))
447		return r535_instmem_new(&nv50_instmem, device, type, inst, pimem);
448
449	return nv50_instmem_new_(&nv50_instmem, device, type, inst, pimem);
450}