crc.c

  1// SPDX-License-Identifier: MIT
  2#include <linux/debugfs.h>
  3#include <linux/string.h>
  4
  5#include <drm/drm_crtc.h>
  6#include <drm/drm_atomic_helper.h>
  7#include <drm/drm_vblank.h>
  8#include <drm/drm_vblank_work.h>
  9
 10#include <nvif/class.h>
 11#include <nvif/cl0002.h>
 12#include <nvif/timer.h>
 13
 14#include <nvhw/class/cl907d.h>
 15
 16#include "nouveau_drv.h"
 17#include "core.h"
 18#include "head.h"
 19#include "wndw.h"
 20#include "handles.h"
 21#include "crc.h"
 22
 23static const char * const nv50_crc_sources[] = {
 24	[NV50_CRC_SOURCE_NONE] = "none",
 25	[NV50_CRC_SOURCE_AUTO] = "auto",
 26	[NV50_CRC_SOURCE_RG] = "rg",
 27	[NV50_CRC_SOURCE_OUTP_ACTIVE] = "outp-active",
 28	[NV50_CRC_SOURCE_OUTP_COMPLETE] = "outp-complete",
 29	[NV50_CRC_SOURCE_OUTP_INACTIVE] = "outp-inactive",
 30};
 31
 32static int nv50_crc_parse_source(const char *buf, enum nv50_crc_source *s)
 33{
 34	int i;
 35
 36	if (!buf) {
 37		*s = NV50_CRC_SOURCE_NONE;
 38		return 0;
 39	}
 40
 41	i = match_string(nv50_crc_sources, ARRAY_SIZE(nv50_crc_sources), buf);
 42	if (i < 0)
 43		return i;
 44
 45	*s = i;
 46	return 0;
 47}
 48
 49int
 50nv50_crc_verify_source(struct drm_crtc *crtc, const char *source_name,
 51		       size_t *values_cnt)
 52{
 53	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
 54	enum nv50_crc_source source;
 55
 56	if (nv50_crc_parse_source(source_name, &source) < 0) {
 57		NV_DEBUG(drm, "unknown source %s\n", source_name);
 58		return -EINVAL;
 59	}
 60
 61	*values_cnt = 1;
 62	return 0;
 63}
 64
 65const char *const *nv50_crc_get_sources(struct drm_crtc *crtc, size_t *count)
 66{
 67	*count = ARRAY_SIZE(nv50_crc_sources);
 68	return nv50_crc_sources;
 69}
 70
 71static void
 72nv50_crc_program_ctx(struct nv50_head *head,
 73		     struct nv50_crc_notifier_ctx *ctx)
 74{
 75	struct nv50_disp *disp = nv50_disp(head->base.base.dev);
 76	struct nv50_core *core = disp->core;
 77	u32 interlock[NV50_DISP_INTERLOCK__SIZE] = { 0 };
 78
 79	core->func->crc->set_ctx(head, ctx);
 80	core->func->update(core, interlock, false);
 81}
 82
 83static void nv50_crc_ctx_flip_work(struct kthread_work *base)
 84{
 85	struct drm_vblank_work *work = to_drm_vblank_work(base);
 86	struct nv50_crc *crc = container_of(work, struct nv50_crc, flip_work);
 87	struct nv50_head *head = container_of(crc, struct nv50_head, crc);
 88	struct drm_crtc *crtc = &head->base.base;
 89	struct drm_device *dev = crtc->dev;
 90	struct nv50_disp *disp = nv50_disp(dev);
 91	const uint64_t start_vbl = drm_crtc_vblank_count(crtc);
 92	uint64_t end_vbl;
 93	u8 new_idx = crc->ctx_idx ^ 1;
 94
 95	/*
 96	 * We don't want to accidentally wait for longer then the vblank, so
 97	 * try again for the next vblank if we don't grab the lock
 98	 */
 99	if (!mutex_trylock(&disp->mutex)) {
100		drm_dbg_kms(dev, "Lock contended, delaying CRC ctx flip for %s\n", crtc->name);
101		drm_vblank_work_schedule(work, start_vbl + 1, true);
102		return;
103	}
104
105	drm_dbg_kms(dev, "Flipping notifier ctx for %s (%d -> %d)\n",
106		    crtc->name, crc->ctx_idx, new_idx);
107
108	nv50_crc_program_ctx(head, NULL);
109	nv50_crc_program_ctx(head, &crc->ctx[new_idx]);
110	mutex_unlock(&disp->mutex);
111
112	end_vbl = drm_crtc_vblank_count(crtc);
113	if (unlikely(end_vbl != start_vbl))
114		NV_ERROR(nouveau_drm(dev),
115			 "Failed to flip CRC context on %s on time (%llu > %llu)\n",
116			 crtc->name, end_vbl, start_vbl);
117
118	spin_lock_irq(&crc->lock);
119	crc->ctx_changed = true;
120	spin_unlock_irq(&crc->lock);
121}
122
123static inline void nv50_crc_reset_ctx(struct nv50_crc_notifier_ctx *ctx)
124{
125	memset_io(ctx->mem.object.map.ptr, 0, ctx->mem.object.map.size);
126}
127
128static void
129nv50_crc_get_entries(struct nv50_head *head,
130		     const struct nv50_crc_func *func,
131		     enum nv50_crc_source source)
132{
133	struct drm_crtc *crtc = &head->base.base;
134	struct nv50_crc *crc = &head->crc;
135	u32 output_crc;
136
137	while (crc->entry_idx < func->num_entries) {
138		/*
139		 * While Nvidia's documentation says CRCs are written on each
140		 * subsequent vblank after being enabled, in practice they
141		 * aren't written immediately.
142		 */
143		output_crc = func->get_entry(head, &crc->ctx[crc->ctx_idx],
144					     source, crc->entry_idx);
145		if (!output_crc)
146			return;
147
148		drm_crtc_add_crc_entry(crtc, true, crc->frame, &output_crc);
149		crc->frame++;
150		crc->entry_idx++;
151	}
152}
153
154void nv50_crc_handle_vblank(struct nv50_head *head)
155{
156	struct drm_crtc *crtc = &head->base.base;
157	struct nv50_crc *crc = &head->crc;
158	const struct nv50_crc_func *func =
159		nv50_disp(head->base.base.dev)->core->func->crc;
160	struct nv50_crc_notifier_ctx *ctx;
161	bool need_reschedule = false;
162
163	if (!func)
164		return;
165
166	/*
167	 * We don't lose events if we aren't able to report CRCs until the
168	 * next vblank, so only report CRCs if the locks we need aren't
169	 * contended to prevent missing an actual vblank event
170	 */
171	if (!spin_trylock(&crc->lock))
172		return;
173
174	if (!crc->src)
175		goto out;
176
177	ctx = &crc->ctx[crc->ctx_idx];
178	if (crc->ctx_changed && func->ctx_finished(head, ctx)) {
179		nv50_crc_get_entries(head, func, crc->src);
180
181		crc->ctx_idx ^= 1;
182		crc->entry_idx = 0;
183		crc->ctx_changed = false;
184
185		/*
186		 * Unfortunately when notifier contexts are changed during CRC
187		 * capture, we will inevitably lose the CRC entry for the
188		 * frame where the hardware actually latched onto the first
189		 * UPDATE. According to Nvidia's hardware engineers, there's
190		 * no workaround for this.
191		 *
192		 * Now, we could try to be smart here and calculate the number
193		 * of missed CRCs based on audit timestamps, but those were
194		 * removed starting with volta. Since we always flush our
195		 * updates back-to-back without waiting, we'll just be
196		 * optimistic and assume we always miss exactly one frame.
197		 */
198		drm_dbg_kms(head->base.base.dev,
199			    "Notifier ctx flip for head-%d finished, lost CRC for frame %llu\n",
200			    head->base.index, crc->frame);
201		crc->frame++;
202
203		nv50_crc_reset_ctx(ctx);
204		need_reschedule = true;
205	}
206
207	nv50_crc_get_entries(head, func, crc->src);
208
209	if (need_reschedule)
210		drm_vblank_work_schedule(&crc->flip_work,
211					 drm_crtc_vblank_count(crtc)
212					 + crc->flip_threshold
213					 - crc->entry_idx,
214					 true);
215
216out:
217	spin_unlock(&crc->lock);
218}
219
220static void nv50_crc_wait_ctx_finished(struct nv50_head *head,
221				       const struct nv50_crc_func *func,
222				       struct nv50_crc_notifier_ctx *ctx)
223{
224	struct drm_device *dev = head->base.base.dev;
225	struct nouveau_drm *drm = nouveau_drm(dev);
226	s64 ret;
227
228	ret = nvif_msec(&drm->client.device, 50,
229			if (func->ctx_finished(head, ctx)) break;);
230	if (ret == -ETIMEDOUT)
231		NV_ERROR(drm,
232			 "CRC notifier ctx for head %d not finished after 50ms\n",
233			 head->base.index);
234	else if (ret)
235		NV_ATOMIC(drm,
236			  "CRC notifier ctx for head-%d finished after %lldns\n",
237			  head->base.index, ret);
238}
239
240void nv50_crc_atomic_stop_reporting(struct drm_atomic_state *state)
241{
242	struct drm_crtc_state *crtc_state;
243	struct drm_crtc *crtc;
244	int i;
245
246	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
247		struct nv50_head *head = nv50_head(crtc);
248		struct nv50_head_atom *asyh = nv50_head_atom(crtc_state);
249		struct nv50_crc *crc = &head->crc;
250
251		if (!asyh->clr.crc)
252			continue;
253
254		spin_lock_irq(&crc->lock);
255		crc->src = NV50_CRC_SOURCE_NONE;
256		spin_unlock_irq(&crc->lock);
257
258		drm_crtc_vblank_put(crtc);
259		drm_vblank_work_cancel_sync(&crc->flip_work);
260
261		NV_ATOMIC(nouveau_drm(crtc->dev),
262			  "CRC reporting on vblank for head-%d disabled\n",
263			  head->base.index);
264
265		/* CRC generation is still enabled in hw, we'll just report
266		 * any remaining CRC entries ourselves after it gets disabled
267		 * in hardware
268		 */
269	}
270}
271
272void nv50_crc_atomic_init_notifier_contexts(struct drm_atomic_state *state)
273{
274	struct drm_crtc_state *new_crtc_state;
275	struct drm_crtc *crtc;
276	int i;
277
278	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
279		struct nv50_head *head = nv50_head(crtc);
280		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
281		struct nv50_crc *crc = &head->crc;
282		int i;
283
284		if (!asyh->set.crc)
285			continue;
286
287		crc->entry_idx = 0;
288		crc->ctx_changed = false;
289		for (i = 0; i < ARRAY_SIZE(crc->ctx); i++)
290			nv50_crc_reset_ctx(&crc->ctx[i]);
291	}
292}
293
294void nv50_crc_atomic_release_notifier_contexts(struct drm_atomic_state *state)
295{
296	const struct nv50_crc_func *func =
297		nv50_disp(state->dev)->core->func->crc;
298	struct drm_crtc_state *new_crtc_state;
299	struct drm_crtc *crtc;
300	int i;
301
302	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
303		struct nv50_head *head = nv50_head(crtc);
304		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
305		struct nv50_crc *crc = &head->crc;
306		struct nv50_crc_notifier_ctx *ctx = &crc->ctx[crc->ctx_idx];
307
308		if (!asyh->clr.crc)
309			continue;
310
311		if (crc->ctx_changed) {
312			nv50_crc_wait_ctx_finished(head, func, ctx);
313			ctx = &crc->ctx[crc->ctx_idx ^ 1];
314		}
315		nv50_crc_wait_ctx_finished(head, func, ctx);
316	}
317}
318
319void nv50_crc_atomic_start_reporting(struct drm_atomic_state *state)
320{
321	struct drm_crtc_state *crtc_state;
322	struct drm_crtc *crtc;
323	int i;
324
325	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
326		struct nv50_head *head = nv50_head(crtc);
327		struct nv50_head_atom *asyh = nv50_head_atom(crtc_state);
328		struct nv50_crc *crc = &head->crc;
329		u64 vbl_count;
330
331		if (!asyh->set.crc)
332			continue;
333
334		drm_crtc_vblank_get(crtc);
335
336		spin_lock_irq(&crc->lock);
337		vbl_count = drm_crtc_vblank_count(crtc);
338		crc->frame = vbl_count;
339		crc->src = asyh->crc.src;
340		drm_vblank_work_schedule(&crc->flip_work,
341					 vbl_count + crc->flip_threshold,
342					 true);
343		spin_unlock_irq(&crc->lock);
344
345		NV_ATOMIC(nouveau_drm(crtc->dev),
346			  "CRC reporting on vblank for head-%d enabled\n",
347			  head->base.index);
348	}
349}
350
351int nv50_crc_atomic_check_head(struct nv50_head *head,
352			       struct nv50_head_atom *asyh,
353			       struct nv50_head_atom *armh)
354{
355	struct nv50_atom *atom = nv50_atom(asyh->state.state);
356	bool changed = armh->crc.src != asyh->crc.src;
357
358	if (!armh->crc.src && !asyh->crc.src) {
359		asyh->set.crc = false;
360		asyh->clr.crc = false;
361		return 0;
362	}
363
364	if (drm_atomic_crtc_needs_modeset(&asyh->state) || changed) {
365		asyh->clr.crc = armh->crc.src && armh->state.active;
366		asyh->set.crc = asyh->crc.src && asyh->state.active;
367		if (changed)
368			asyh->set.or |= armh->or.crc_raster !=
369					asyh->or.crc_raster;
370
371		if (asyh->clr.crc && asyh->set.crc)
372			atom->flush_disable = true;
373	} else {
374		asyh->set.crc = false;
375		asyh->clr.crc = false;
376	}
377
378	return 0;
379}
380
381void nv50_crc_atomic_check_outp(struct nv50_atom *atom)
382{
383	struct drm_crtc *crtc;
384	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
385	int i;
386
387	if (atom->flush_disable)
388		return;
389
390	for_each_oldnew_crtc_in_state(&atom->state, crtc, old_crtc_state,
391				      new_crtc_state, i) {
392		struct nv50_head_atom *armh = nv50_head_atom(old_crtc_state);
393		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
394		struct nv50_outp_atom *outp_atom;
395		struct nouveau_encoder *outp;
396		struct drm_encoder *encoder, *enc;
397
398		enc = nv50_head_atom_get_encoder(armh);
399		if (!enc)
400			continue;
401
402		outp = nv50_real_outp(enc);
403		if (!outp)
404			continue;
405
406		encoder = &outp->base.base;
407
408		if (!asyh->clr.crc)
409			continue;
410
411		/*
412		 * Re-programming ORs can't be done in the same flush as
413		 * disabling CRCs
414		 */
415		list_for_each_entry(outp_atom, &atom->outp, head) {
416			if (outp_atom->encoder == encoder) {
417				if (outp_atom->set.mask) {
418					atom->flush_disable = true;
419					return;
420				} else {
421					break;
422				}
423			}
424		}
425	}
426}
427
428static enum nv50_crc_source_type
429nv50_crc_source_type(struct nouveau_encoder *outp,
430		     enum nv50_crc_source source)
431{
432	struct dcb_output *dcbe = outp->dcb;
433
434	switch (source) {
435	case NV50_CRC_SOURCE_NONE: return NV50_CRC_SOURCE_TYPE_NONE;
436	case NV50_CRC_SOURCE_RG:   return NV50_CRC_SOURCE_TYPE_RG;
437	default:		   break;
438	}
439
440	if (dcbe->location != DCB_LOC_ON_CHIP)
441		return NV50_CRC_SOURCE_TYPE_PIOR;
442
443	switch (dcbe->type) {
444	case DCB_OUTPUT_DP:	return NV50_CRC_SOURCE_TYPE_SF;
445	case DCB_OUTPUT_ANALOG:	return NV50_CRC_SOURCE_TYPE_DAC;
446	default:		return NV50_CRC_SOURCE_TYPE_SOR;
447	}
448}
449
450void nv50_crc_atomic_set(struct nv50_head *head,
451			 struct nv50_head_atom *asyh)
452{
453	struct drm_crtc *crtc = &head->base.base;
454	struct drm_device *dev = crtc->dev;
455	struct nv50_crc *crc = &head->crc;
456	const struct nv50_crc_func *func = nv50_disp(dev)->core->func->crc;
457	struct nouveau_encoder *outp;
458	struct drm_encoder *encoder;
459
460	encoder = nv50_head_atom_get_encoder(asyh);
461	if (!encoder)
462		return;
463
464	outp = nv50_real_outp(encoder);
465	if (!outp)
466		return;
467
468	func->set_src(head, outp->outp.or.id, nv50_crc_source_type(outp, asyh->crc.src),
469		      &crc->ctx[crc->ctx_idx]);
470}
471
472void nv50_crc_atomic_clr(struct nv50_head *head)
473{
474	const struct nv50_crc_func *func =
475		nv50_disp(head->base.base.dev)->core->func->crc;
476
477	func->set_src(head, 0, NV50_CRC_SOURCE_TYPE_NONE, NULL);
478}
479
480static inline int
481nv50_crc_raster_type(enum nv50_crc_source source)
482{
483	switch (source) {
484	case NV50_CRC_SOURCE_NONE:
485	case NV50_CRC_SOURCE_AUTO:
486	case NV50_CRC_SOURCE_RG:
487	case NV50_CRC_SOURCE_OUTP_ACTIVE:
488		return NV907D_HEAD_SET_CONTROL_OUTPUT_RESOURCE_CRC_MODE_ACTIVE_RASTER;
489	case NV50_CRC_SOURCE_OUTP_COMPLETE:
490		return NV907D_HEAD_SET_CONTROL_OUTPUT_RESOURCE_CRC_MODE_COMPLETE_RASTER;
491	case NV50_CRC_SOURCE_OUTP_INACTIVE:
492		return NV907D_HEAD_SET_CONTROL_OUTPUT_RESOURCE_CRC_MODE_NON_ACTIVE_RASTER;
493	}
494
495	return 0;
496}
497
498/* We handle mapping the memory for CRC notifiers ourselves, since each
499 * notifier needs it's own handle
500 */
501static inline int
502nv50_crc_ctx_init(struct nv50_head *head, struct nvif_mmu *mmu,
503		  struct nv50_crc_notifier_ctx *ctx, size_t len, int idx)
504{
505	struct nv50_core *core = nv50_disp(head->base.base.dev)->core;
506	int ret;
507
508	ret = nvif_mem_ctor_map(mmu, "kmsCrcNtfy", NVIF_MEM_VRAM, len, &ctx->mem);
509	if (ret)
510		return ret;
511
512	/* No CTXDMAs on Blackwell. */
513	if (core->chan.base.user.oclass >= GB202_DISP_CORE_CHANNEL_DMA)
514		return 0;
515
516	ret = nvif_object_ctor(&core->chan.base.user, "kmsCrcNtfyCtxDma",
517			       NV50_DISP_HANDLE_CRC_CTX(head, idx),
518			       NV_DMA_IN_MEMORY,
519			       &(struct nv_dma_v0) {
520					.target = NV_DMA_V0_TARGET_VRAM,
521					.access = NV_DMA_V0_ACCESS_RDWR,
522					.start = ctx->mem.addr,
523					.limit =  ctx->mem.addr
524						+ ctx->mem.size - 1,
525			       }, sizeof(struct nv_dma_v0),
526			       &ctx->ntfy);
527	if (ret)
528		goto fail_fini;
529
530	return 0;
531
532fail_fini:
533	nvif_mem_dtor(&ctx->mem);
534	return ret;
535}
536
537static inline void
538nv50_crc_ctx_fini(struct nv50_crc_notifier_ctx *ctx)
539{
540	nvif_object_dtor(&ctx->ntfy);
541	nvif_mem_dtor(&ctx->mem);
542}
543
544int nv50_crc_set_source(struct drm_crtc *crtc, const char *source_str)
545{
546	struct drm_device *dev = crtc->dev;
547	struct drm_atomic_state *state;
548	struct drm_modeset_acquire_ctx ctx;
549	struct nv50_head *head = nv50_head(crtc);
550	struct nv50_crc *crc = &head->crc;
551	const struct nv50_crc_func *func = nv50_disp(dev)->core->func->crc;
552	struct nvif_mmu *mmu = &nouveau_drm(dev)->client.mmu;
553	struct nv50_head_atom *asyh;
554	struct drm_crtc_state *crtc_state;
555	enum nv50_crc_source source;
556	int ret = 0, ctx_flags = 0, i;
557
558	ret = nv50_crc_parse_source(source_str, &source);
559	if (ret)
560		return ret;
561
562	/*
563	 * Since we don't want the user to accidentally interrupt us as we're
564	 * disabling CRCs
565	 */
566	if (source)
567		ctx_flags |= DRM_MODESET_ACQUIRE_INTERRUPTIBLE;
568	drm_modeset_acquire_init(&ctx, ctx_flags);
569
570	state = drm_atomic_state_alloc(dev);
571	if (!state) {
572		ret = -ENOMEM;
573		goto out_acquire_fini;
574	}
575	state->acquire_ctx = &ctx;
576
577	if (source) {
578		for (i = 0; i < ARRAY_SIZE(head->crc.ctx); i++) {
579			ret = nv50_crc_ctx_init(head, mmu, &crc->ctx[i],
580						func->notifier_len, i);
581			if (ret)
582				goto out_ctx_fini;
583		}
584	}
585
586retry:
587	crtc_state = drm_atomic_get_crtc_state(state, &head->base.base);
588	if (IS_ERR(crtc_state)) {
589		ret = PTR_ERR(crtc_state);
590		if (ret == -EDEADLK)
591			goto deadlock;
592		else if (ret)
593			goto out_drop_locks;
594	}
595	asyh = nv50_head_atom(crtc_state);
596	asyh->crc.src = source;
597	asyh->or.crc_raster = nv50_crc_raster_type(source);
598
599	ret = drm_atomic_commit(state);
600	if (ret == -EDEADLK)
601		goto deadlock;
602	else if (ret)
603		goto out_drop_locks;
604
605	if (!source) {
606		/*
607		 * If the user specified a custom flip threshold through
608		 * debugfs, reset it
609		 */
610		crc->flip_threshold = func->flip_threshold;
611	}
612
613out_drop_locks:
614	drm_modeset_drop_locks(&ctx);
615out_ctx_fini:
616	if (!source || ret) {
617		for (i = 0; i < ARRAY_SIZE(crc->ctx); i++)
618			nv50_crc_ctx_fini(&crc->ctx[i]);
619	}
620	drm_atomic_state_put(state);
621out_acquire_fini:
622	drm_modeset_acquire_fini(&ctx);
623	return ret;
624
625deadlock:
626	drm_atomic_state_clear(state);
627	drm_modeset_backoff(&ctx);
628	goto retry;
629}
630
631static int
632nv50_crc_debugfs_flip_threshold_get(struct seq_file *m, void *data)
633{
634	struct nv50_head *head = m->private;
635	struct drm_crtc *crtc = &head->base.base;
636	struct nv50_crc *crc = &head->crc;
637	int ret;
638
639	ret = drm_modeset_lock_single_interruptible(&crtc->mutex);
640	if (ret)
641		return ret;
642
643	seq_printf(m, "%d\n", crc->flip_threshold);
644
645	drm_modeset_unlock(&crtc->mutex);
646	return ret;
647}
648
649static int
650nv50_crc_debugfs_flip_threshold_open(struct inode *inode, struct file *file)
651{
652	return single_open(file, nv50_crc_debugfs_flip_threshold_get,
653			   inode->i_private);
654}
655
656static ssize_t
657nv50_crc_debugfs_flip_threshold_set(struct file *file,
658				    const char __user *ubuf, size_t len,
659				    loff_t *offp)
660{
661	struct seq_file *m = file->private_data;
662	struct nv50_head *head = m->private;
663	struct nv50_head_atom *armh;
664	struct drm_crtc *crtc = &head->base.base;
665	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
666	struct nv50_crc *crc = &head->crc;
667	const struct nv50_crc_func *func =
668		nv50_disp(crtc->dev)->core->func->crc;
669	int value, ret;
670
671	ret = kstrtoint_from_user(ubuf, len, 10, &value);
672	if (ret)
673		return ret;
674
675	if (value > func->flip_threshold)
676		return -EINVAL;
677	else if (value == -1)
678		value = func->flip_threshold;
679	else if (value < -1)
680		return -EINVAL;
681
682	ret = drm_modeset_lock_single_interruptible(&crtc->mutex);
683	if (ret)
684		return ret;
685
686	armh = nv50_head_atom(crtc->state);
687	if (armh->crc.src) {
688		ret = -EBUSY;
689		goto out;
690	}
691
692	NV_DEBUG(drm,
693		 "Changing CRC flip threshold for next capture on head-%d to %d\n",
694		 head->base.index, value);
695	crc->flip_threshold = value;
696	ret = len;
697
698out:
699	drm_modeset_unlock(&crtc->mutex);
700	return ret;
701}
702
703static const struct file_operations nv50_crc_flip_threshold_fops = {
704	.owner = THIS_MODULE,
705	.open = nv50_crc_debugfs_flip_threshold_open,
706	.read = seq_read,
707	.write = nv50_crc_debugfs_flip_threshold_set,
708	.release = single_release,
709};
710
711int nv50_head_crc_late_register(struct nv50_head *head)
712{
713	struct drm_crtc *crtc = &head->base.base;
714	const struct nv50_crc_func *func =
715		nv50_disp(crtc->dev)->core->func->crc;
716	struct dentry *root;
717
718	if (!func || !crtc->debugfs_entry)
719		return 0;
720
721	root = debugfs_create_dir("nv_crc", crtc->debugfs_entry);
722	debugfs_create_file("flip_threshold", 0644, root, head,
723			    &nv50_crc_flip_threshold_fops);
724
725	return 0;
726}
727
728static inline void
729nv50_crc_init_head(struct nv50_disp *disp, const struct nv50_crc_func *func,
730		   struct nv50_head *head)
731{
732	struct nv50_crc *crc = &head->crc;
733
734	crc->flip_threshold = func->flip_threshold;
735	spin_lock_init(&crc->lock);
736	drm_vblank_work_init(&crc->flip_work, &head->base.base,
737			     nv50_crc_ctx_flip_work);
738}
739
740void nv50_crc_init(struct drm_device *dev)
741{
742	struct nv50_disp *disp = nv50_disp(dev);
743	struct drm_crtc *crtc;
744	const struct nv50_crc_func *func = disp->core->func->crc;
745
746	if (!func)
747		return;
748
749	drm_for_each_crtc(crtc, dev)
750		nv50_crc_init_head(disp, func, nv50_head(crtc));
751}