disp.c

   1/*
   2 * Copyright 2011 Red Hat Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Ben Skeggs
  23 */
  24#include "disp.h"
  25#include "atom.h"
  26#include "core.h"
  27#include "head.h"
  28#include "wndw.h"
  29#include "handles.h"
  30
  31#include <linux/backlight.h>
  32#include <linux/dma-mapping.h>
  33#include <linux/hdmi.h>
  34#include <linux/component.h>
  35#include <linux/iopoll.h>
  36
  37#include <drm/display/drm_dp_helper.h>
  38#include <drm/display/drm_scdc_helper.h>
  39#include <drm/drm_atomic.h>
  40#include <drm/drm_atomic_helper.h>
  41#include <drm/drm_edid.h>
  42#include <drm/drm_eld.h>
  43#include <drm/drm_fb_helper.h>
  44#include <drm/drm_fixed.h>
  45#include <drm/drm_probe_helper.h>
  46#include <drm/drm_vblank.h>
  47
  48#include <nvif/push507c.h>
  49
  50#include <nvif/class.h>
  51#include <nvif/cl0002.h>
  52#include <nvif/event.h>
  53#include <nvif/if0012.h>
  54#include <nvif/if0014.h>
  55#include <nvif/timer.h>
  56
  57#include <nvhw/class/cl507c.h>
  58#include <nvhw/class/cl507d.h>
  59#include <nvhw/class/cl837d.h>
  60#include <nvhw/class/cl887d.h>
  61#include <nvhw/class/cl907d.h>
  62#include <nvhw/class/cl917d.h>
  63
  64#include "nouveau_drv.h"
  65#include "nouveau_dma.h"
  66#include "nouveau_gem.h"
  67#include "nouveau_connector.h"
  68#include "nouveau_encoder.h"
  69#include "nouveau_fence.h"
  70#include "nv50_display.h"
  71
  72/******************************************************************************
  73 * EVO channel
  74 *****************************************************************************/
  75
  76static int
  77nv50_chan_create(struct nvif_device *device, struct nvif_object *disp,
  78		 const s32 *oclass, u8 head, void *data, u32 size,
  79		 struct nv50_chan *chan)
  80{
  81	struct nvif_sclass *sclass;
  82	int ret, i, n;
  83
  84	chan->device = device;
  85
  86	ret = n = nvif_object_sclass_get(disp, &sclass);
  87	if (ret < 0)
  88		return ret;
  89
  90	while (oclass[0]) {
  91		for (i = 0; i < n; i++) {
  92			if (sclass[i].oclass == oclass[0]) {
  93				ret = nvif_object_ctor(disp, "kmsChan", 0,
  94						       oclass[0], data, size,
  95						       &chan->user);
  96				if (ret == 0) {
  97					ret = nvif_object_map(&chan->user, NULL, 0);
  98					if (ret)
  99						nvif_object_dtor(&chan->user);
 100				}
 101				nvif_object_sclass_put(&sclass);
 102				return ret;
 103			}
 104		}
 105		oclass++;
 106	}
 107
 108	nvif_object_sclass_put(&sclass);
 109	return -ENOSYS;
 110}
 111
 112static void
 113nv50_chan_destroy(struct nv50_chan *chan)
 114{
 115	nvif_object_dtor(&chan->user);
 116}
 117
 118/******************************************************************************
 119 * DMA EVO channel
 120 *****************************************************************************/
 121
 122void
 123nv50_dmac_destroy(struct nv50_dmac *dmac)
 124{
 125	nvif_object_dtor(&dmac->vram);
 126	nvif_object_dtor(&dmac->sync);
 127
 128	nv50_chan_destroy(&dmac->base);
 129
 130	nvif_mem_dtor(&dmac->push.mem);
 131}
 132
 133static void
 134nv50_dmac_kick(struct nvif_push *push)
 135{
 136	struct nv50_dmac *dmac = container_of(push, typeof(*dmac), push);
 137
 138	dmac->cur = push->cur - (u32 __iomem *)dmac->push.mem.object.map.ptr;
 139	if (dmac->put != dmac->cur) {
 140		/* Push buffer fetches are not coherent with BAR1, we need to ensure
 141		 * writes have been flushed right through to VRAM before writing PUT.
 142		 */
 143		if (dmac->push.mem.type & NVIF_MEM_VRAM) {
 144			struct nvif_device *device = dmac->base.device;
 145			nvif_wr32(&device->object, 0x070000, 0x00000001);
 146			nvif_msec(device, 2000,
 147				if (!(nvif_rd32(&device->object, 0x070000) & 0x00000002))
 148					break;
 149			);
 150		}
 151
 152		NVIF_WV32(&dmac->base.user, NV507C, PUT, PTR, dmac->cur);
 153		dmac->put = dmac->cur;
 154	}
 155
 156	push->bgn = push->cur;
 157}
 158
 159static int
 160nv50_dmac_free(struct nv50_dmac *dmac)
 161{
 162	u32 get = NVIF_RV32(&dmac->base.user, NV507C, GET, PTR);
 163	if (get > dmac->cur) /* NVIDIA stay 5 away from GET, do the same. */
 164		return get - dmac->cur - 5;
 165	return dmac->max - dmac->cur;
 166}
 167
 168static int
 169nv50_dmac_wind(struct nv50_dmac *dmac)
 170{
 171	/* Wait for GET to depart from the beginning of the push buffer to
 172	 * prevent writing PUT == GET, which would be ignored by HW.
 173	 */
 174	u32 get = NVIF_RV32(&dmac->base.user, NV507C, GET, PTR);
 175	if (get == 0) {
 176		/* Corner-case, HW idle, but non-committed work pending. */
 177		if (dmac->put == 0)
 178			nv50_dmac_kick(&dmac->push);
 179
 180		if (nvif_msec(dmac->base.device, 2000,
 181			if (NVIF_TV32(&dmac->base.user, NV507C, GET, PTR, >, 0))
 182				break;
 183		) < 0)
 184			return -ETIMEDOUT;
 185	}
 186
 187	PUSH_RSVD(&dmac->push, PUSH_JUMP(&dmac->push, 0));
 188	dmac->cur = 0;
 189	return 0;
 190}
 191
 192static int
 193nv50_dmac_wait(struct nvif_push *push, u32 size)
 194{
 195	struct nv50_dmac *dmac = container_of(push, typeof(*dmac), push);
 196	int free;
 197
 198	if (WARN_ON(size > dmac->max))
 199		return -EINVAL;
 200
 201	dmac->cur = push->cur - (u32 __iomem *)dmac->push.mem.object.map.ptr;
 202	if (dmac->cur + size >= dmac->max) {
 203		int ret = nv50_dmac_wind(dmac);
 204		if (ret)
 205			return ret;
 206
 207		push->cur = dmac->push.mem.object.map.ptr;
 208		push->cur = push->cur + dmac->cur;
 209		nv50_dmac_kick(push);
 210	}
 211
 212	if (nvif_msec(dmac->base.device, 2000,
 213		if ((free = nv50_dmac_free(dmac)) >= size)
 214			break;
 215	) < 0) {
 216		WARN_ON(1);
 217		return -ETIMEDOUT;
 218	}
 219
 220	push->bgn = dmac->push.mem.object.map.ptr;
 221	push->bgn = push->bgn + dmac->cur;
 222	push->cur = push->bgn;
 223	push->end = push->cur + free;
 224	return 0;
 225}
 226
 227MODULE_PARM_DESC(kms_vram_pushbuf, "Place EVO/NVD push buffers in VRAM (default: auto)");
 228static int nv50_dmac_vram_pushbuf = -1;
 229module_param_named(kms_vram_pushbuf, nv50_dmac_vram_pushbuf, int, 0400);
 230
 231int
 232nv50_dmac_create(struct nouveau_drm *drm,
 233		 const s32 *oclass, u8 head, void *data, u32 size, s64 syncbuf,
 234		 struct nv50_dmac *dmac)
 235{
 236	struct nvif_device *device = &drm->device;
 237	struct nvif_object *disp = &drm->display->disp.object;
 238	struct nvif_disp_chan_v0 *args = data;
 239	u8 type = NVIF_MEM_COHERENT;
 240	int ret;
 241
 242	/* Pascal added support for 47-bit physical addresses, but some
 243	 * parts of EVO still only accept 40-bit PAs.
 244	 *
 245	 * To avoid issues on systems with large amounts of RAM, and on
 246	 * systems where an IOMMU maps pages at a high address, we need
 247	 * to allocate push buffers in VRAM instead.
 248	 *
 249	 * This appears to match NVIDIA's behaviour on Pascal.
 250	 */
 251	if ((nv50_dmac_vram_pushbuf > 0) ||
 252	    (nv50_dmac_vram_pushbuf < 0 && device->info.family == NV_DEVICE_INFO_V0_PASCAL))
 253		type |= NVIF_MEM_VRAM;
 254
 255	ret = nvif_mem_ctor_map(&drm->mmu, "kmsChanPush", type, 0x1000, &dmac->push.mem);
 256	if (ret)
 257		return ret;
 258
 259	dmac->push.wait = nv50_dmac_wait;
 260	dmac->push.kick = nv50_dmac_kick;
 261	dmac->push.bgn = dmac->push.mem.object.map.ptr;
 262	dmac->push.cur = dmac->push.bgn;
 263	dmac->push.end = dmac->push.bgn;
 264	dmac->max = 0x1000/4 - 1;
 265
 266	/* EVO channels are affected by a HW bug where the last 12 DWORDs
 267	 * of the push buffer aren't able to be used safely.
 268	 */
 269	if (disp->oclass < GV100_DISP)
 270		dmac->max -= 12;
 271
 272	args->pushbuf = nvif_handle(&dmac->push.mem.object);
 273
 274	ret = nv50_chan_create(device, disp, oclass, head, data, size,
 275			       &dmac->base);
 276	if (ret)
 277		return ret;
 278
 279	if (syncbuf < 0)
 280		return 0;
 281
 282	/* No CTXDMAs on Blackwell. */
 283	if (disp->oclass >= GB202_DISP) {
 284		/* "handle != NULL_HANDLE" is used to determine enable status
 285		 * in a number of places, so fill in some fake object handles.
 286		 */
 287		dmac->sync.handle = NV50_DISP_HANDLE_SYNCBUF;
 288		dmac->vram.handle = NV50_DISP_HANDLE_VRAM;
 289		return 0;
 290	}
 291
 292	ret = nvif_object_ctor(&dmac->base.user, "kmsSyncCtxDma", NV50_DISP_HANDLE_SYNCBUF,
 293			       NV_DMA_IN_MEMORY,
 294			       &(struct nv_dma_v0) {
 295					.target = NV_DMA_V0_TARGET_VRAM,
 296					.access = NV_DMA_V0_ACCESS_RDWR,
 297					.start = syncbuf + 0x0000,
 298					.limit = syncbuf + 0x0fff,
 299			       }, sizeof(struct nv_dma_v0),
 300			       &dmac->sync);
 301	if (ret)
 302		return ret;
 303
 304	ret = nvif_object_ctor(&dmac->base.user, "kmsVramCtxDma", NV50_DISP_HANDLE_VRAM,
 305			       NV_DMA_IN_MEMORY,
 306			       &(struct nv_dma_v0) {
 307					.target = NV_DMA_V0_TARGET_VRAM,
 308					.access = NV_DMA_V0_ACCESS_RDWR,
 309					.start = 0,
 310					.limit = device->info.ram_user - 1,
 311			       }, sizeof(struct nv_dma_v0),
 312			       &dmac->vram);
 313	if (ret)
 314		return ret;
 315
 316	return ret;
 317}
 318
 319/******************************************************************************
 320 * Output path helpers
 321 *****************************************************************************/
 322static void
 323nv50_outp_dump_caps(struct nouveau_drm *drm,
 324		    struct nouveau_encoder *outp)
 325{
 326	NV_DEBUG(drm, "%s caps: dp_interlace=%d\n",
 327		 outp->base.base.name, outp->caps.dp_interlace);
 328}
 329
 330static int
 331nv50_outp_atomic_check_view(struct drm_encoder *encoder,
 332			    struct drm_crtc_state *crtc_state,
 333			    struct drm_connector_state *conn_state,
 334			    struct drm_display_mode *native_mode)
 335{
 336	struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
 337	struct drm_display_mode *mode = &crtc_state->mode;
 338	struct drm_connector *connector = conn_state->connector;
 339	struct nouveau_conn_atom *asyc = nouveau_conn_atom(conn_state);
 340	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 341
 342	NV_ATOMIC(drm, "%s atomic_check\n", encoder->name);
 343	asyc->scaler.full = false;
 344	if (!native_mode)
 345		return 0;
 346
 347	if (asyc->scaler.mode == DRM_MODE_SCALE_NONE) {
 348		switch (connector->connector_type) {
 349		case DRM_MODE_CONNECTOR_LVDS:
 350		case DRM_MODE_CONNECTOR_eDP:
 351			/* Don't force scaler for EDID modes with
 352			 * same size as the native one (e.g. different
 353			 * refresh rate)
 354			 */
 355			if (mode->hdisplay == native_mode->hdisplay &&
 356			    mode->vdisplay == native_mode->vdisplay &&
 357			    mode->type & DRM_MODE_TYPE_DRIVER)
 358				break;
 359			mode = native_mode;
 360			asyc->scaler.full = true;
 361			break;
 362		default:
 363			break;
 364		}
 365	} else {
 366		mode = native_mode;
 367	}
 368
 369	if (!drm_mode_equal(adjusted_mode, mode)) {
 370		drm_mode_copy(adjusted_mode, mode);
 371		crtc_state->mode_changed = true;
 372	}
 373
 374	return 0;
 375}
 376
 377static void
 378nv50_outp_atomic_fix_depth(struct drm_encoder *encoder, struct drm_crtc_state *crtc_state)
 379{
 380	struct nv50_head_atom *asyh = nv50_head_atom(crtc_state);
 381	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 382	struct drm_display_mode *mode = &asyh->state.adjusted_mode;
 383	unsigned int max_rate, mode_rate;
 384
 385	switch (nv_encoder->dcb->type) {
 386	case DCB_OUTPUT_DP:
 387		max_rate = nv_encoder->dp.link_nr * nv_encoder->dp.link_bw;
 388
 389		/* we don't support more than 10 anyway */
 390		asyh->or.bpc = min_t(u8, asyh->or.bpc, 10);
 391
 392		/* reduce the bpc until it works out */
 393		while (asyh->or.bpc > 6) {
 394			mode_rate = DIV_ROUND_UP(mode->clock * asyh->or.bpc * 3, 8);
 395			if (mode_rate <= max_rate)
 396				break;
 397
 398			asyh->or.bpc -= 2;
 399		}
 400		break;
 401	default:
 402		break;
 403	}
 404}
 405
 406static int
 407nv50_outp_atomic_check(struct drm_encoder *encoder,
 408		       struct drm_crtc_state *crtc_state,
 409		       struct drm_connector_state *conn_state)
 410{
 411	struct drm_connector *connector = conn_state->connector;
 412	struct nouveau_connector *nv_connector = nouveau_connector(connector);
 413	struct nv50_head_atom *asyh = nv50_head_atom(crtc_state);
 414	int ret;
 415
 416	ret = nv50_outp_atomic_check_view(encoder, crtc_state, conn_state,
 417					  nv_connector->native_mode);
 418	if (ret)
 419		return ret;
 420
 421	if (crtc_state->mode_changed || crtc_state->connectors_changed)
 422		asyh->or.bpc = connector->display_info.bpc;
 423
 424	/* We might have to reduce the bpc */
 425	nv50_outp_atomic_fix_depth(encoder, crtc_state);
 426
 427	return 0;
 428}
 429
 430struct nouveau_connector *
 431nv50_outp_get_new_connector(struct drm_atomic_state *state, struct nouveau_encoder *outp)
 432{
 433	struct drm_connector *connector;
 434	struct drm_connector_state *connector_state;
 435	struct drm_encoder *encoder = to_drm_encoder(outp);
 436	int i;
 437
 438	for_each_new_connector_in_state(state, connector, connector_state, i) {
 439		if (connector_state->best_encoder == encoder)
 440			return nouveau_connector(connector);
 441	}
 442
 443	return NULL;
 444}
 445
 446struct nouveau_connector *
 447nv50_outp_get_old_connector(struct drm_atomic_state *state, struct nouveau_encoder *outp)
 448{
 449	struct drm_connector *connector;
 450	struct drm_connector_state *connector_state;
 451	struct drm_encoder *encoder = to_drm_encoder(outp);
 452	int i;
 453
 454	for_each_old_connector_in_state(state, connector, connector_state, i) {
 455		if (connector_state->best_encoder == encoder)
 456			return nouveau_connector(connector);
 457	}
 458
 459	return NULL;
 460}
 461
 462static struct nouveau_crtc *
 463nv50_outp_get_new_crtc(const struct drm_atomic_state *state, const struct nouveau_encoder *outp)
 464{
 465	struct drm_crtc *crtc;
 466	struct drm_crtc_state *crtc_state;
 467	const u32 mask = drm_encoder_mask(&outp->base.base);
 468	int i;
 469
 470	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
 471		if (crtc_state->encoder_mask & mask)
 472			return nouveau_crtc(crtc);
 473	}
 474
 475	return NULL;
 476}
 477
 478/******************************************************************************
 479 * DAC
 480 *****************************************************************************/
 481static void
 482nv50_dac_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *state)
 483{
 484	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 485	struct nv50_core *core = nv50_disp(encoder->dev)->core;
 486	const u32 ctrl = NVDEF(NV507D, DAC_SET_CONTROL, OWNER, NONE);
 487
 488	core->func->dac->ctrl(core, nv_encoder->outp.or.id, ctrl, NULL);
 489	nv_encoder->crtc = NULL;
 490}
 491
 492static void
 493nv50_dac_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *state)
 494{
 495	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 496	struct nouveau_crtc *nv_crtc = nv50_outp_get_new_crtc(state, nv_encoder);
 497	struct nv50_head_atom *asyh =
 498		nv50_head_atom(drm_atomic_get_new_crtc_state(state, &nv_crtc->base));
 499	struct nv50_core *core = nv50_disp(encoder->dev)->core;
 500	u32 ctrl = 0;
 501
 502	switch (nv_crtc->index) {
 503	case 0: ctrl |= NVDEF(NV507D, DAC_SET_CONTROL, OWNER, HEAD0); break;
 504	case 1: ctrl |= NVDEF(NV507D, DAC_SET_CONTROL, OWNER, HEAD1); break;
 505	case 2: ctrl |= NVDEF(NV907D, DAC_SET_CONTROL, OWNER_MASK, HEAD2); break;
 506	case 3: ctrl |= NVDEF(NV907D, DAC_SET_CONTROL, OWNER_MASK, HEAD3); break;
 507	default:
 508		WARN_ON(1);
 509		break;
 510	}
 511
 512	ctrl |= NVDEF(NV507D, DAC_SET_CONTROL, PROTOCOL, RGB_CRT);
 513
 514	if (!nvif_outp_acquired(&nv_encoder->outp))
 515		nvif_outp_acquire_dac(&nv_encoder->outp);
 516
 517	core->func->dac->ctrl(core, nv_encoder->outp.or.id, ctrl, asyh);
 518	asyh->or.depth = 0;
 519
 520	nv_encoder->crtc = &nv_crtc->base;
 521}
 522
 523static enum drm_connector_status
 524nv50_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector)
 525{
 526	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 527	u32 loadval;
 528	int ret;
 529
 530	loadval = nouveau_drm(encoder->dev)->vbios.dactestval;
 531	if (loadval == 0)
 532		loadval = 340;
 533
 534	ret = nvif_outp_load_detect(&nv_encoder->outp, loadval);
 535	if (ret <= 0)
 536		return connector_status_disconnected;
 537
 538	return connector_status_connected;
 539}
 540
 541static const struct drm_encoder_helper_funcs
 542nv50_dac_help = {
 543	.atomic_check = nv50_outp_atomic_check,
 544	.atomic_enable = nv50_dac_atomic_enable,
 545	.atomic_disable = nv50_dac_atomic_disable,
 546	.detect = nv50_dac_detect
 547};
 548
 549static void
 550nv50_dac_destroy(struct drm_encoder *encoder)
 551{
 552	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 553
 554	nvif_outp_dtor(&nv_encoder->outp);
 555
 556	drm_encoder_cleanup(encoder);
 557	kfree(encoder);
 558}
 559
 560static const struct drm_encoder_funcs
 561nv50_dac_func = {
 562	.destroy = nv50_dac_destroy,
 563};
 564
 565static int
 566nv50_dac_create(struct nouveau_encoder *nv_encoder)
 567{
 568	struct drm_connector *connector = &nv_encoder->conn->base;
 569	struct nouveau_drm *drm = nouveau_drm(connector->dev);
 570	struct nvkm_i2c *i2c = nvxx_i2c(drm);
 571	struct nvkm_i2c_bus *bus;
 572	struct drm_encoder *encoder;
 573	struct dcb_output *dcbe = nv_encoder->dcb;
 574	int type = DRM_MODE_ENCODER_DAC;
 575
 576	bus = nvkm_i2c_bus_find(i2c, dcbe->i2c_index);
 577	if (bus)
 578		nv_encoder->i2c = &bus->i2c;
 579
 580	encoder = to_drm_encoder(nv_encoder);
 581	drm_encoder_init(connector->dev, encoder, &nv50_dac_func, type,
 582			 "dac-%04x-%04x", dcbe->hasht, dcbe->hashm);
 583	drm_encoder_helper_add(encoder, &nv50_dac_help);
 584
 585	drm_connector_attach_encoder(connector, encoder);
 586	return 0;
 587}
 588
 589/*
 590 * audio component binding for ELD notification
 591 */
 592static void
 593nv50_audio_component_eld_notify(struct drm_audio_component *acomp, int port,
 594				int dev_id)
 595{
 596	if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify)
 597		acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr,
 598						 port, dev_id);
 599}
 600
 601static int
 602nv50_audio_component_get_eld(struct device *kdev, int port, int dev_id,
 603			     bool *enabled, unsigned char *buf, int max_bytes)
 604{
 605	struct nouveau_drm *drm = dev_get_drvdata(kdev);
 606	struct drm_encoder *encoder;
 607	struct nouveau_encoder *nv_encoder;
 608	struct nouveau_crtc *nv_crtc;
 609	int ret = 0;
 610
 611	*enabled = false;
 612
 613	mutex_lock(&drm->audio.lock);
 614
 615	drm_for_each_encoder(encoder, drm->dev) {
 616		struct nouveau_connector *nv_connector = NULL;
 617
 618		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST)
 619			continue; /* TODO */
 620
 621		nv_encoder = nouveau_encoder(encoder);
 622		nv_connector = nv_encoder->conn;
 623		nv_crtc = nouveau_crtc(nv_encoder->crtc);
 624
 625		if (!nv_crtc || nv_encoder->outp.or.id != port || nv_crtc->index != dev_id)
 626			continue;
 627
 628		*enabled = nv_encoder->audio.enabled;
 629		if (*enabled) {
 630			ret = drm_eld_size(nv_connector->base.eld);
 631			memcpy(buf, nv_connector->base.eld,
 632			       min(max_bytes, ret));
 633		}
 634		break;
 635	}
 636
 637	mutex_unlock(&drm->audio.lock);
 638
 639	return ret;
 640}
 641
 642static const struct drm_audio_component_ops nv50_audio_component_ops = {
 643	.get_eld = nv50_audio_component_get_eld,
 644};
 645
 646static int
 647nv50_audio_component_bind(struct device *kdev, struct device *hda_kdev,
 648			  void *data)
 649{
 650	struct nouveau_drm *drm = dev_get_drvdata(kdev);
 651	struct drm_audio_component *acomp = data;
 652
 653	if (WARN_ON(!device_link_add(hda_kdev, kdev, DL_FLAG_STATELESS)))
 654		return -ENOMEM;
 655
 656	drm_modeset_lock_all(drm->dev);
 657	acomp->ops = &nv50_audio_component_ops;
 658	acomp->dev = kdev;
 659	drm->audio.component = acomp;
 660	drm_modeset_unlock_all(drm->dev);
 661	return 0;
 662}
 663
 664static void
 665nv50_audio_component_unbind(struct device *kdev, struct device *hda_kdev,
 666			    void *data)
 667{
 668	struct nouveau_drm *drm = dev_get_drvdata(kdev);
 669	struct drm_audio_component *acomp = data;
 670
 671	drm_modeset_lock_all(drm->dev);
 672	drm->audio.component = NULL;
 673	acomp->ops = NULL;
 674	acomp->dev = NULL;
 675	drm_modeset_unlock_all(drm->dev);
 676}
 677
 678static const struct component_ops nv50_audio_component_bind_ops = {
 679	.bind   = nv50_audio_component_bind,
 680	.unbind = nv50_audio_component_unbind,
 681};
 682
 683static void
 684nv50_audio_component_init(struct nouveau_drm *drm)
 685{
 686	if (component_add(drm->dev->dev, &nv50_audio_component_bind_ops))
 687		return;
 688
 689	drm->audio.component_registered = true;
 690	mutex_init(&drm->audio.lock);
 691}
 692
 693static void
 694nv50_audio_component_fini(struct nouveau_drm *drm)
 695{
 696	if (!drm->audio.component_registered)
 697		return;
 698
 699	component_del(drm->dev->dev, &nv50_audio_component_bind_ops);
 700	drm->audio.component_registered = false;
 701	mutex_destroy(&drm->audio.lock);
 702}
 703
 704/******************************************************************************
 705 * Audio
 706 *****************************************************************************/
 707static bool
 708nv50_audio_supported(struct drm_encoder *encoder)
 709{
 710	struct nv50_disp *disp = nv50_disp(encoder->dev);
 711
 712	if (disp->disp->object.oclass <= GT200_DISP ||
 713	    disp->disp->object.oclass == GT206_DISP)
 714		return false;
 715
 716	if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
 717		struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 718
 719		switch (nv_encoder->dcb->type) {
 720		case DCB_OUTPUT_TMDS:
 721		case DCB_OUTPUT_DP:
 722			break;
 723		default:
 724			return false;
 725		}
 726	}
 727
 728	return true;
 729}
 730
 731static void
 732nv50_audio_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc)
 733{
 734	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 735	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 736	struct nvif_outp *outp = &nv_encoder->outp;
 737
 738	if (!nv50_audio_supported(encoder))
 739		return;
 740
 741	mutex_lock(&drm->audio.lock);
 742	if (nv_encoder->audio.enabled) {
 743		nv_encoder->audio.enabled = false;
 744		nvif_outp_hda_eld(&nv_encoder->outp, nv_crtc->index, NULL, 0);
 745	}
 746	mutex_unlock(&drm->audio.lock);
 747
 748	nv50_audio_component_eld_notify(drm->audio.component, outp->or.id, nv_crtc->index);
 749}
 750
 751static void
 752nv50_audio_enable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc,
 753		  struct nouveau_connector *nv_connector, struct drm_atomic_state *state,
 754		  struct drm_display_mode *mode)
 755{
 756	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 757	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 758	struct nvif_outp *outp = &nv_encoder->outp;
 759
 760	if (!nv50_audio_supported(encoder) || !nv_connector->base.display_info.has_audio)
 761		return;
 762
 763	mutex_lock(&drm->audio.lock);
 764
 765	nvif_outp_hda_eld(&nv_encoder->outp, nv_crtc->index, nv_connector->base.eld,
 766			  drm_eld_size(nv_connector->base.eld));
 767	nv_encoder->audio.enabled = true;
 768
 769	mutex_unlock(&drm->audio.lock);
 770
 771	nv50_audio_component_eld_notify(drm->audio.component, outp->or.id, nv_crtc->index);
 772}
 773
 774/******************************************************************************
 775 * HDMI
 776 *****************************************************************************/
 777static void
 778nv50_hdmi_enable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc,
 779		 struct nouveau_connector *nv_connector, struct drm_atomic_state *state,
 780		 struct drm_display_mode *mode, bool hda)
 781{
 782	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 783	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 784	struct drm_hdmi_info *hdmi = &nv_connector->base.display_info.hdmi;
 785	union hdmi_infoframe infoframe = { 0 };
 786	const u8 rekey = 56; /* binary driver, and tegra, constant */
 787	u32 max_ac_packet;
 788	DEFINE_RAW_FLEX(struct nvif_outp_infoframe_v0, args, data, 17);
 789	const u8 data_len = __member_size(args->data);
 790	int ret, size;
 791
 792	max_ac_packet  = mode->htotal - mode->hdisplay;
 793	max_ac_packet -= rekey;
 794	max_ac_packet -= 18; /* constant from tegra */
 795	max_ac_packet /= 32;
 796
 797	if (nv_encoder->i2c && hdmi->scdc.scrambling.supported) {
 798		const bool high_tmds_clock_ratio = mode->clock > 340000;
 799		u8 scdc;
 800
 801		ret = drm_scdc_readb(nv_encoder->i2c, SCDC_TMDS_CONFIG, &scdc);
 802		if (ret < 0) {
 803			NV_ERROR(drm, "Failure to read SCDC_TMDS_CONFIG: %d\n", ret);
 804			return;
 805		}
 806
 807		scdc &= ~(SCDC_TMDS_BIT_CLOCK_RATIO_BY_40 | SCDC_SCRAMBLING_ENABLE);
 808		if (high_tmds_clock_ratio || hdmi->scdc.scrambling.low_rates)
 809			scdc |= SCDC_SCRAMBLING_ENABLE;
 810		if (high_tmds_clock_ratio)
 811			scdc |= SCDC_TMDS_BIT_CLOCK_RATIO_BY_40;
 812
 813		ret = drm_scdc_writeb(nv_encoder->i2c, SCDC_TMDS_CONFIG, scdc);
 814		if (ret < 0)
 815			NV_ERROR(drm, "Failure to write SCDC_TMDS_CONFIG = 0x%02x: %d\n",
 816				 scdc, ret);
 817	}
 818
 819	ret = nvif_outp_hdmi(&nv_encoder->outp, nv_crtc->index, true, max_ac_packet, rekey,
 820			     mode->clock, hdmi->scdc.supported, hdmi->scdc.scrambling.supported,
 821			     hdmi->scdc.scrambling.low_rates);
 822	if (ret)
 823		return;
 824
 825	/* AVI InfoFrame. */
 826	args->version = 0;
 827	args->head = nv_crtc->index;
 828
 829	if (!drm_hdmi_avi_infoframe_from_display_mode(&infoframe.avi, &nv_connector->base, mode)) {
 830		drm_hdmi_avi_infoframe_quant_range(&infoframe.avi, &nv_connector->base, mode,
 831						   HDMI_QUANTIZATION_RANGE_FULL);
 832
 833		size = hdmi_infoframe_pack(&infoframe, args->data, data_len);
 834	} else {
 835		size = 0;
 836	}
 837
 838	nvif_outp_infoframe(&nv_encoder->outp, NVIF_OUTP_INFOFRAME_V0_AVI, args, size);
 839
 840	/* Vendor InfoFrame. */
 841	memset(args->data, 0, data_len);
 842	if (!drm_hdmi_vendor_infoframe_from_display_mode(&infoframe.vendor.hdmi,
 843							 &nv_connector->base, mode))
 844		size = hdmi_infoframe_pack(&infoframe, args->data, data_len);
 845	else
 846		size = 0;
 847
 848	nvif_outp_infoframe(&nv_encoder->outp, NVIF_OUTP_INFOFRAME_V0_VSI, args, size);
 849
 850	nv_encoder->hdmi.enabled = true;
 851}
 852
 853/******************************************************************************
 854 * MST
 855 *****************************************************************************/
 856#define nv50_mstm(p) container_of((p), struct nv50_mstm, mgr)
 857#define nv50_mstc(p) container_of((p), struct nv50_mstc, connector)
 858#define nv50_msto(p) container_of((p), struct nv50_msto, encoder)
 859
 860struct nv50_mstc {
 861	struct nv50_mstm *mstm;
 862	struct drm_dp_mst_port *port;
 863	struct drm_connector connector;
 864
 865	struct drm_display_mode *native;
 866	struct edid *edid;
 867};
 868
 869struct nv50_msto {
 870	struct drm_encoder encoder;
 871
 872	/* head is statically assigned on msto creation */
 873	struct nv50_head *head;
 874	struct nv50_mstc *mstc;
 875	bool disabled;
 876	bool enabled;
 877
 878	u32 display_id;
 879};
 880
 881struct nouveau_encoder *nv50_real_outp(struct drm_encoder *encoder)
 882{
 883	struct nv50_msto *msto;
 884
 885	if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST)
 886		return nouveau_encoder(encoder);
 887
 888	msto = nv50_msto(encoder);
 889	if (!msto->mstc)
 890		return NULL;
 891	return msto->mstc->mstm->outp;
 892}
 893
 894static void
 895nv50_msto_cleanup(struct drm_atomic_state *state,
 896		  struct drm_dp_mst_topology_state *new_mst_state,
 897		  struct drm_dp_mst_topology_mgr *mgr,
 898		  struct nv50_msto *msto)
 899{
 900	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
 901	struct drm_dp_mst_atomic_payload *new_payload =
 902		drm_atomic_get_mst_payload_state(new_mst_state, msto->mstc->port);
 903	struct drm_dp_mst_topology_state *old_mst_state =
 904		drm_atomic_get_old_mst_topology_state(state, mgr);
 905	const struct drm_dp_mst_atomic_payload *old_payload =
 906		drm_atomic_get_mst_payload_state(old_mst_state, msto->mstc->port);
 907	struct nv50_mstc *mstc = msto->mstc;
 908	struct nv50_mstm *mstm = mstc->mstm;
 909
 910	NV_ATOMIC(drm, "%s: msto cleanup\n", msto->encoder.name);
 911
 912	if (msto->disabled) {
 913		if (msto->head->func->display_id) {
 914			nvif_outp_dp_mst_id_put(&mstm->outp->outp, msto->display_id);
 915			msto->display_id = 0;
 916		}
 917
 918		msto->mstc = NULL;
 919		msto->disabled = false;
 920		drm_dp_remove_payload_part2(mgr, new_mst_state, old_payload, new_payload);
 921	} else if (msto->enabled) {
 922		drm_dp_add_payload_part2(mgr, new_payload);
 923		msto->enabled = false;
 924	}
 925}
 926
 927static void
 928nv50_msto_prepare(struct drm_atomic_state *state,
 929		  struct drm_dp_mst_topology_state *mst_state,
 930		  struct drm_dp_mst_topology_mgr *mgr,
 931		  struct nv50_msto *msto)
 932{
 933	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
 934	struct nv50_mstc *mstc = msto->mstc;
 935	struct nv50_mstm *mstm = mstc->mstm;
 936	struct drm_dp_mst_atomic_payload *payload;
 937	int ret = 0;
 938
 939	NV_ATOMIC(drm, "%s: msto prepare\n", msto->encoder.name);
 940
 941	payload = drm_atomic_get_mst_payload_state(mst_state, mstc->port);
 942
 943	if (msto->disabled) {
 944		drm_dp_remove_payload_part1(mgr, mst_state, payload);
 945		nvif_outp_dp_mst_vcpi(&mstm->outp->outp, msto->head->base.index, 0, 0, 0, 0);
 946		ret = 1;
 947	} else {
 948		if (msto->enabled)
 949			ret = drm_dp_add_payload_part1(mgr, mst_state, payload);
 950	}
 951
 952	if (ret == 0) {
 953		nvif_outp_dp_mst_vcpi(&mstm->outp->outp, msto->head->base.index,
 954				      payload->vc_start_slot, payload->time_slots,
 955				      payload->pbn,
 956				      payload->time_slots * dfixed_trunc(mst_state->pbn_div));
 957	} else {
 958		nvif_outp_dp_mst_vcpi(&mstm->outp->outp, msto->head->base.index, 0, 0, 0, 0);
 959	}
 960}
 961
 962static int
 963nv50_msto_atomic_check(struct drm_encoder *encoder,
 964		       struct drm_crtc_state *crtc_state,
 965		       struct drm_connector_state *conn_state)
 966{
 967	struct drm_atomic_state *state = crtc_state->state;
 968	struct drm_connector *connector = conn_state->connector;
 969	struct drm_dp_mst_topology_state *mst_state;
 970	struct nv50_mstc *mstc = nv50_mstc(connector);
 971	struct nv50_mstm *mstm = mstc->mstm;
 972	struct nv50_head_atom *asyh = nv50_head_atom(crtc_state);
 973	int slots;
 974	int ret;
 975
 976	ret = nv50_outp_atomic_check_view(encoder, crtc_state, conn_state,
 977					  mstc->native);
 978	if (ret)
 979		return ret;
 980
 981	if (!drm_atomic_crtc_needs_modeset(crtc_state))
 982		return 0;
 983
 984	/*
 985	 * When restoring duplicated states, we need to make sure that the bw
 986	 * remains the same and avoid recalculating it, as the connector's bpc
 987	 * may have changed after the state was duplicated
 988	 */
 989	if (!state->duplicated) {
 990		const int clock = crtc_state->adjusted_mode.clock;
 991
 992		asyh->or.bpc = connector->display_info.bpc;
 993		asyh->dp.pbn = drm_dp_calc_pbn_mode(clock, asyh->or.bpc * 3 << 4);
 994	}
 995
 996	mst_state = drm_atomic_get_mst_topology_state(state, &mstm->mgr);
 997	if (IS_ERR(mst_state))
 998		return PTR_ERR(mst_state);
 999
1000	if (!mst_state->pbn_div.full) {
1001		struct nouveau_encoder *outp = mstc->mstm->outp;
1002
1003		mst_state->pbn_div = drm_dp_get_vc_payload_bw(outp->dp.link_bw, outp->dp.link_nr);
1004	}
1005
1006	slots = drm_dp_atomic_find_time_slots(state, &mstm->mgr, mstc->port, asyh->dp.pbn);
1007	if (slots < 0)
1008		return slots;
1009
1010	asyh->dp.tu = slots;
1011
1012	return 0;
1013}
1014
1015static u8
1016nv50_dp_bpc_to_depth(unsigned int bpc)
1017{
1018	switch (bpc) {
1019	case  6: return NV837D_SOR_SET_CONTROL_PIXEL_DEPTH_BPP_18_444;
1020	case  8: return NV837D_SOR_SET_CONTROL_PIXEL_DEPTH_BPP_24_444;
1021	case 10:
1022	default: return NV837D_SOR_SET_CONTROL_PIXEL_DEPTH_BPP_30_444;
1023	}
1024}
1025
1026static void
1027nv50_msto_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *state)
1028{
1029	struct nv50_msto *msto = nv50_msto(encoder);
1030	struct nv50_head *head = msto->head;
1031	struct nv50_head_atom *asyh =
1032		nv50_head_atom(drm_atomic_get_new_crtc_state(state, &head->base.base));
1033	struct nv50_mstc *mstc = NULL;
1034	struct nv50_mstm *mstm = NULL;
1035	struct drm_connector *connector;
1036	struct drm_connector_list_iter conn_iter;
1037	u8 proto;
1038
1039	drm_connector_list_iter_begin(encoder->dev, &conn_iter);
1040	drm_for_each_connector_iter(connector, &conn_iter) {
1041		if (connector->state->best_encoder == &msto->encoder) {
1042			mstc = nv50_mstc(connector);
1043			mstm = mstc->mstm;
1044			break;
1045		}
1046	}
1047	drm_connector_list_iter_end(&conn_iter);
1048
1049	if (WARN_ON(!mstc))
1050		return;
1051
1052	if (!mstm->links++) {
1053		nvif_outp_acquire_sor(&mstm->outp->outp, false /*TODO: MST audio... */);
1054		nouveau_dp_train(mstm->outp, true, 0, 0);
1055	}
1056
1057	if (head->func->display_id) {
1058		if (!WARN_ON(nvif_outp_dp_mst_id_get(&mstm->outp->outp, &msto->display_id)))
1059			head->func->display_id(head, msto->display_id);
1060	}
1061
1062	if (mstm->outp->outp.or.link & 1)
1063		proto = NV917D_SOR_SET_CONTROL_PROTOCOL_DP_A;
1064	else
1065		proto = NV917D_SOR_SET_CONTROL_PROTOCOL_DP_B;
1066
1067	mstm->outp->update(mstm->outp, head->base.index, asyh, proto,
1068			   nv50_dp_bpc_to_depth(asyh->or.bpc));
1069
1070	msto->mstc = mstc;
1071	msto->enabled = true;
1072	mstm->modified = true;
1073}
1074
1075static void
1076nv50_msto_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *state)
1077{
1078	struct nv50_msto *msto = nv50_msto(encoder);
1079	struct nv50_mstc *mstc = msto->mstc;
1080	struct nv50_mstm *mstm = mstc->mstm;
1081
1082	if (msto->head->func->display_id)
1083		msto->head->func->display_id(msto->head, 0);
1084
1085	mstm->outp->update(mstm->outp, msto->head->base.index, NULL, 0, 0);
1086	mstm->modified = true;
1087	if (!--mstm->links)
1088		mstm->disabled = true;
1089	msto->disabled = true;
1090}
1091
1092static const struct drm_encoder_helper_funcs
1093nv50_msto_help = {
1094	.atomic_disable = nv50_msto_atomic_disable,
1095	.atomic_enable = nv50_msto_atomic_enable,
1096	.atomic_check = nv50_msto_atomic_check,
1097};
1098
1099static void
1100nv50_msto_destroy(struct drm_encoder *encoder)
1101{
1102	struct nv50_msto *msto = nv50_msto(encoder);
1103	drm_encoder_cleanup(&msto->encoder);
1104	kfree(msto);
1105}
1106
1107static const struct drm_encoder_funcs
1108nv50_msto = {
1109	.destroy = nv50_msto_destroy,
1110};
1111
1112static struct nv50_msto *
1113nv50_msto_new(struct drm_device *dev, struct nv50_head *head, int id)
1114{
1115	struct nv50_msto *msto;
1116	int ret;
1117
1118	msto = kzalloc(sizeof(*msto), GFP_KERNEL);
1119	if (!msto)
1120		return ERR_PTR(-ENOMEM);
1121
1122	ret = drm_encoder_init(dev, &msto->encoder, &nv50_msto,
1123			       DRM_MODE_ENCODER_DPMST, "mst-%d", id);
1124	if (ret) {
1125		kfree(msto);
1126		return ERR_PTR(ret);
1127	}
1128
1129	drm_encoder_helper_add(&msto->encoder, &nv50_msto_help);
1130	msto->encoder.possible_crtcs = drm_crtc_mask(&head->base.base);
1131	msto->head = head;
1132	return msto;
1133}
1134
1135static struct drm_encoder *
1136nv50_mstc_atomic_best_encoder(struct drm_connector *connector,
1137			      struct drm_atomic_state *state)
1138{
1139	struct drm_connector_state *connector_state = drm_atomic_get_new_connector_state(state,
1140											 connector);
1141	struct nv50_mstc *mstc = nv50_mstc(connector);
1142	struct drm_crtc *crtc = connector_state->crtc;
1143
1144	if (!(mstc->mstm->outp->dcb->heads & drm_crtc_mask(crtc)))
1145		return NULL;
1146
1147	return &nv50_head(crtc)->msto->encoder;
1148}
1149
1150static enum drm_mode_status
1151nv50_mstc_mode_valid(struct drm_connector *connector,
1152		     const struct drm_display_mode *mode)
1153{
1154	struct nv50_mstc *mstc = nv50_mstc(connector);
1155	struct nouveau_encoder *outp = mstc->mstm->outp;
1156
1157	/* TODO: calculate the PBN from the dotclock and validate against the
1158	 * MSTB's max possible PBN
1159	 */
1160
1161	return nv50_dp_mode_valid(outp, mode, NULL);
1162}
1163
1164static int
1165nv50_mstc_get_modes(struct drm_connector *connector)
1166{
1167	struct nv50_mstc *mstc = nv50_mstc(connector);
1168	int ret = 0;
1169
1170	mstc->edid = drm_dp_mst_get_edid(&mstc->connector, mstc->port->mgr, mstc->port);
1171	drm_connector_update_edid_property(&mstc->connector, mstc->edid);
1172	if (mstc->edid)
1173		ret = drm_add_edid_modes(&mstc->connector, mstc->edid);
1174
1175	/*
1176	 * XXX: Since we don't use HDR in userspace quite yet, limit the bpc
1177	 * to 8 to save bandwidth on the topology. In the future, we'll want
1178	 * to properly fix this by dynamically selecting the highest possible
1179	 * bpc that would fit in the topology
1180	 */
1181	if (connector->display_info.bpc)
1182		connector->display_info.bpc =
1183			clamp(connector->display_info.bpc, 6U, 8U);
1184	else
1185		connector->display_info.bpc = 8;
1186
1187	if (mstc->native)
1188		drm_mode_destroy(mstc->connector.dev, mstc->native);
1189	mstc->native = nouveau_conn_native_mode(&mstc->connector);
1190	return ret;
1191}
1192
1193static int
1194nv50_mstc_atomic_check(struct drm_connector *connector,
1195		       struct drm_atomic_state *state)
1196{
1197	struct nv50_mstc *mstc = nv50_mstc(connector);
1198	struct drm_dp_mst_topology_mgr *mgr = &mstc->mstm->mgr;
1199
1200	return drm_dp_atomic_release_time_slots(state, mgr, mstc->port);
1201}
1202
1203static int
1204nv50_mstc_detect(struct drm_connector *connector,
1205		 struct drm_modeset_acquire_ctx *ctx, bool force)
1206{
1207	struct nv50_mstc *mstc = nv50_mstc(connector);
1208	int ret;
1209
1210	if (drm_connector_is_unregistered(connector))
1211		return connector_status_disconnected;
1212
1213	ret = pm_runtime_get_sync(connector->dev->dev);
1214	if (ret < 0 && ret != -EACCES) {
1215		pm_runtime_put_autosuspend(connector->dev->dev);
1216		return connector_status_disconnected;
1217	}
1218
1219	ret = drm_dp_mst_detect_port(connector, ctx, mstc->port->mgr,
1220				     mstc->port);
1221	if (ret != connector_status_connected)
1222		goto out;
1223
1224out:
1225	pm_runtime_mark_last_busy(connector->dev->dev);
1226	pm_runtime_put_autosuspend(connector->dev->dev);
1227	return ret;
1228}
1229
1230static const struct drm_connector_helper_funcs
1231nv50_mstc_help = {
1232	.get_modes = nv50_mstc_get_modes,
1233	.mode_valid = nv50_mstc_mode_valid,
1234	.atomic_best_encoder = nv50_mstc_atomic_best_encoder,
1235	.atomic_check = nv50_mstc_atomic_check,
1236	.detect_ctx = nv50_mstc_detect,
1237};
1238
1239static void
1240nv50_mstc_destroy(struct drm_connector *connector)
1241{
1242	struct nv50_mstc *mstc = nv50_mstc(connector);
1243
1244	drm_connector_cleanup(&mstc->connector);
1245	drm_dp_mst_put_port_malloc(mstc->port);
1246
1247	kfree(mstc);
1248}
1249
1250static const struct drm_connector_funcs
1251nv50_mstc = {
1252	.reset = nouveau_conn_reset,
1253	.fill_modes = drm_helper_probe_single_connector_modes,
1254	.destroy = nv50_mstc_destroy,
1255	.atomic_duplicate_state = nouveau_conn_atomic_duplicate_state,
1256	.atomic_destroy_state = nouveau_conn_atomic_destroy_state,
1257	.atomic_set_property = nouveau_conn_atomic_set_property,
1258	.atomic_get_property = nouveau_conn_atomic_get_property,
1259};
1260
1261static int
1262nv50_mstc_new(struct nv50_mstm *mstm, struct drm_dp_mst_port *port,
1263	      const char *path, struct nv50_mstc **pmstc)
1264{
1265	struct drm_device *dev = mstm->outp->base.base.dev;
1266	struct drm_crtc *crtc;
1267	struct nv50_mstc *mstc;
1268	int ret;
1269
1270	if (!(mstc = *pmstc = kzalloc(sizeof(*mstc), GFP_KERNEL)))
1271		return -ENOMEM;
1272	mstc->mstm = mstm;
1273	mstc->port = port;
1274
1275	ret = drm_connector_dynamic_init(dev, &mstc->connector, &nv50_mstc,
1276					 DRM_MODE_CONNECTOR_DisplayPort, NULL);
1277	if (ret) {
1278		kfree(*pmstc);
1279		*pmstc = NULL;
1280		return ret;
1281	}
1282
1283	drm_connector_helper_add(&mstc->connector, &nv50_mstc_help);
1284
1285	mstc->connector.funcs->reset(&mstc->connector);
1286	nouveau_conn_attach_properties(&mstc->connector);
1287
1288	drm_for_each_crtc(crtc, dev) {
1289		if (!(mstm->outp->dcb->heads & drm_crtc_mask(crtc)))
1290			continue;
1291
1292		drm_connector_attach_encoder(&mstc->connector,
1293					     &nv50_head(crtc)->msto->encoder);
1294	}
1295
1296	drm_object_attach_property(&mstc->connector.base, dev->mode_config.path_property, 0);
1297	drm_object_attach_property(&mstc->connector.base, dev->mode_config.tile_property, 0);
1298	drm_connector_set_path_property(&mstc->connector, path);
1299	drm_dp_mst_get_port_malloc(port);
1300	return 0;
1301}
1302
1303static void
1304nv50_mstm_cleanup(struct drm_atomic_state *state,
1305		  struct drm_dp_mst_topology_state *mst_state,
1306		  struct nv50_mstm *mstm)
1307{
1308	struct nouveau_drm *drm = nouveau_drm(mstm->outp->base.base.dev);
1309	struct drm_encoder *encoder;
1310
1311	NV_ATOMIC(drm, "%s: mstm cleanup\n", mstm->outp->base.base.name);
1312	drm_dp_check_act_status(&mstm->mgr);
1313
1314	drm_for_each_encoder(encoder, mstm->outp->base.base.dev) {
1315		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
1316			struct nv50_msto *msto = nv50_msto(encoder);
1317			struct nv50_mstc *mstc = msto->mstc;
1318			if (mstc && mstc->mstm == mstm)
1319				nv50_msto_cleanup(state, mst_state, &mstm->mgr, msto);
1320		}
1321	}
1322
1323	if (mstm->disabled) {
1324		nouveau_dp_power_down(mstm->outp);
1325		nvif_outp_release(&mstm->outp->outp);
1326		mstm->disabled = false;
1327	}
1328
1329	mstm->modified = false;
1330}
1331
1332static void
1333nv50_mstm_prepare(struct drm_atomic_state *state,
1334		  struct drm_dp_mst_topology_state *mst_state,
1335		  struct nv50_mstm *mstm)
1336{
1337	struct nouveau_drm *drm = nouveau_drm(mstm->outp->base.base.dev);
1338	struct drm_encoder *encoder;
1339
1340	NV_ATOMIC(drm, "%s: mstm prepare\n", mstm->outp->base.base.name);
1341
1342	/* Disable payloads first */
1343	drm_for_each_encoder(encoder, mstm->outp->base.base.dev) {
1344		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
1345			struct nv50_msto *msto = nv50_msto(encoder);
1346			struct nv50_mstc *mstc = msto->mstc;
1347			if (mstc && mstc->mstm == mstm && msto->disabled)
1348				nv50_msto_prepare(state, mst_state, &mstm->mgr, msto);
1349		}
1350	}
1351
1352	/* Add payloads for new heads, while also updating the start slots of any unmodified (but
1353	 * active) heads that may have had their VC slots shifted left after the previous step
1354	 */
1355	drm_for_each_encoder(encoder, mstm->outp->base.base.dev) {
1356		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
1357			struct nv50_msto *msto = nv50_msto(encoder);
1358			struct nv50_mstc *mstc = msto->mstc;
1359			if (mstc && mstc->mstm == mstm && !msto->disabled)
1360				nv50_msto_prepare(state, mst_state, &mstm->mgr, msto);
1361		}
1362	}
1363}
1364
1365static struct drm_connector *
1366nv50_mstm_add_connector(struct drm_dp_mst_topology_mgr *mgr,
1367			struct drm_dp_mst_port *port, const char *path)
1368{
1369	struct nv50_mstm *mstm = nv50_mstm(mgr);
1370	struct nv50_mstc *mstc;
1371	int ret;
1372
1373	ret = nv50_mstc_new(mstm, port, path, &mstc);
1374	if (ret)
1375		return NULL;
1376
1377	return &mstc->connector;
1378}
1379
1380static const struct drm_dp_mst_topology_cbs
1381nv50_mstm = {
1382	.add_connector = nv50_mstm_add_connector,
1383};
1384
1385bool
1386nv50_mstm_service(struct nouveau_drm *drm,
1387		  struct nouveau_connector *nv_connector,
1388		  struct nv50_mstm *mstm)
1389{
1390	struct drm_dp_aux *aux = &nv_connector->aux;
1391	bool handled = true, ret = true;
1392	int rc;
1393	u8 esi[8] = {};
1394
1395	while (handled) {
1396		u8 ack[8] = {};
1397
1398		rc = drm_dp_dpcd_read(aux, DP_SINK_COUNT_ESI, esi, 8);
1399		if (rc != 8) {
1400			ret = false;
1401			break;
1402		}
1403
1404		drm_dp_mst_hpd_irq_handle_event(&mstm->mgr, esi, ack, &handled);
1405		if (!handled)
1406			break;
1407
1408		rc = drm_dp_dpcd_writeb(aux, DP_SINK_COUNT_ESI + 1, ack[1]);
1409
1410		if (rc != 1) {
1411			ret = false;
1412			break;
1413		}
1414
1415		drm_dp_mst_hpd_irq_send_new_request(&mstm->mgr);
1416	}
1417
1418	if (!ret)
1419		NV_DEBUG(drm, "Failed to handle ESI on %s: %d\n",
1420			 nv_connector->base.name, rc);
1421
1422	return ret;
1423}
1424
1425void
1426nv50_mstm_remove(struct nv50_mstm *mstm)
1427{
1428	mstm->is_mst = false;
1429	drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, false);
1430}
1431
1432int
1433nv50_mstm_detect(struct nouveau_encoder *outp)
1434{
1435	struct nv50_mstm *mstm = outp->dp.mstm;
1436	struct drm_dp_aux *aux;
1437	int ret;
1438
1439	if (!mstm || !mstm->can_mst)
1440		return 0;
1441
1442	aux = mstm->mgr.aux;
1443
1444	/* Clear any leftover MST state we didn't set ourselves by first
1445	 * disabling MST if it was already enabled
1446	 */
1447	ret = drm_dp_dpcd_writeb(aux, DP_MSTM_CTRL, 0);
1448	if (ret < 0)
1449		return ret;
1450
1451	/* And start enabling */
1452	ret = drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, true);
1453	if (ret)
1454		return ret;
1455
1456	mstm->is_mst = true;
1457	return 1;
1458}
1459
1460static void
1461nv50_mstm_fini(struct nouveau_encoder *outp)
1462{
1463	struct nv50_mstm *mstm = outp->dp.mstm;
1464
1465	if (!mstm)
1466		return;
1467
1468	/* Don't change the MST state of this connector until we've finished
1469	 * resuming, since we can't safely grab hpd_irq_lock in our resume
1470	 * path to protect mstm->is_mst without potentially deadlocking
1471	 */
1472	mutex_lock(&outp->dp.hpd_irq_lock);
1473	mstm->suspended = true;
1474	mutex_unlock(&outp->dp.hpd_irq_lock);
1475
1476	if (mstm->is_mst)
1477		drm_dp_mst_topology_mgr_suspend(&mstm->mgr);
1478}
1479
1480static void
1481nv50_mstm_init(struct nouveau_encoder *outp, bool runtime)
1482{
1483	struct nv50_mstm *mstm = outp->dp.mstm;
1484	int ret = 0;
1485
1486	if (!mstm)
1487		return;
1488
1489	if (mstm->is_mst) {
1490		ret = drm_dp_mst_topology_mgr_resume(&mstm->mgr, !runtime);
1491		if (ret == -1)
1492			nv50_mstm_remove(mstm);
1493	}
1494
1495	mutex_lock(&outp->dp.hpd_irq_lock);
1496	mstm->suspended = false;
1497	mutex_unlock(&outp->dp.hpd_irq_lock);
1498
1499	if (ret == -1)
1500		drm_kms_helper_hotplug_event(mstm->mgr.dev);
1501}
1502
1503static void
1504nv50_mstm_del(struct nv50_mstm **pmstm)
1505{
1506	struct nv50_mstm *mstm = *pmstm;
1507	if (mstm) {
1508		drm_dp_mst_topology_mgr_destroy(&mstm->mgr);
1509		kfree(*pmstm);
1510		*pmstm = NULL;
1511	}
1512}
1513
1514static int
1515nv50_mstm_new(struct nouveau_encoder *outp, struct drm_dp_aux *aux, int aux_max,
1516	      int conn_base_id, struct nv50_mstm **pmstm)
1517{
1518	const int max_payloads = hweight8(outp->dcb->heads);
1519	struct drm_device *dev = outp->base.base.dev;
1520	struct nv50_mstm *mstm;
1521	int ret;
1522
1523	if (!(mstm = *pmstm = kzalloc(sizeof(*mstm), GFP_KERNEL)))
1524		return -ENOMEM;
1525	mstm->outp = outp;
1526	mstm->mgr.cbs = &nv50_mstm;
1527
1528	ret = drm_dp_mst_topology_mgr_init(&mstm->mgr, dev, aux, aux_max,
1529					   max_payloads, conn_base_id);
1530	if (ret)
1531		return ret;
1532
1533	return 0;
1534}
1535
1536/******************************************************************************
1537 * SOR
1538 *****************************************************************************/
1539static void
1540nv50_sor_update(struct nouveau_encoder *nv_encoder, u8 head,
1541		struct nv50_head_atom *asyh, u8 proto, u8 depth)
1542{
1543	struct nv50_disp *disp = nv50_disp(nv_encoder->base.base.dev);
1544	struct nv50_core *core = disp->core;
1545
1546	if (!asyh) {
1547		nv_encoder->ctrl &= ~BIT(head);
1548		if (NVDEF_TEST(nv_encoder->ctrl, NV507D, SOR_SET_CONTROL, OWNER, ==, NONE))
1549			nv_encoder->ctrl = 0;
1550	} else {
1551		nv_encoder->ctrl |= NVVAL(NV507D, SOR_SET_CONTROL, PROTOCOL, proto);
1552		nv_encoder->ctrl |= BIT(head);
1553		asyh->or.depth = depth;
1554	}
1555
1556	core->func->sor->ctrl(core, nv_encoder->outp.or.id, nv_encoder->ctrl, asyh);
1557}
1558
1559/* TODO: Should we extend this to PWM-only backlights?
1560 * As well, should we add a DRM helper for waiting for the backlight to acknowledge
1561 * the panel backlight has been shut off? Intel doesn't seem to do this, and uses a
1562 * fixed time delay from the vbios…
1563 */
1564static void
1565nv50_sor_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *state)
1566{
1567	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
1568	struct nv50_head *head = nv50_head(nv_encoder->crtc);
1569#ifdef CONFIG_DRM_NOUVEAU_BACKLIGHT
1570	struct nouveau_connector *nv_connector = nv50_outp_get_old_connector(state, nv_encoder);
1571	struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
1572	struct nouveau_backlight *backlight = nv_connector->backlight;
1573	struct drm_dp_aux *aux = &nv_connector->aux;
1574	int ret;
1575
1576	if (backlight && backlight->uses_dpcd) {
1577		ret = drm_edp_backlight_disable(aux, &backlight->edp_info);
1578		if (ret < 0)
1579			NV_ERROR(drm, "Failed to disable backlight on [CONNECTOR:%d:%s]: %d\n",
1580				 nv_connector->base.base.id, nv_connector->base.name, ret);
1581	}
1582#endif
1583
1584	if (nv_encoder->dcb->type == DCB_OUTPUT_TMDS && nv_encoder->hdmi.enabled) {
1585		nvif_outp_hdmi(&nv_encoder->outp, head->base.index,
1586			       false, 0, 0, 0, false, false, false);
1587		nv_encoder->hdmi.enabled = false;
1588	}
1589
1590	if (nv_encoder->dcb->type == DCB_OUTPUT_DP)
1591		nouveau_dp_power_down(nv_encoder);
1592
1593	if (head->func->display_id)
1594		head->func->display_id(head, 0);
1595
1596	nv_encoder->update(nv_encoder, head->base.index, NULL, 0, 0);
1597	nv50_audio_disable(encoder, &head->base);
1598	nv_encoder->crtc = NULL;
1599}
1600
1601// common/inc/displayport/displayport.h
1602#define DP_CONFIG_WATERMARK_ADJUST                   2
1603#define DP_CONFIG_WATERMARK_LIMIT                   20
1604#define DP_CONFIG_INCREASED_WATERMARK_ADJUST         8
1605#define DP_CONFIG_INCREASED_WATERMARK_LIMIT         22
1606
1607static bool
1608nv50_sor_dp_watermark_sst(struct nouveau_encoder *outp,
1609			  struct nv50_head *head, struct nv50_head_atom *asyh)
1610{
1611	bool enhancedFraming = outp->dp.dpcd[DP_MAX_LANE_COUNT] & DP_ENHANCED_FRAME_CAP;
1612	u64 minRate = outp->dp.link_bw * 1000;
1613	unsigned tuSize = 64;
1614	unsigned waterMark;
1615	unsigned hBlankSym;
1616	unsigned vBlankSym;
1617	unsigned watermarkAdjust = DP_CONFIG_WATERMARK_ADJUST;
1618	unsigned watermarkMinimum = DP_CONFIG_WATERMARK_LIMIT;
1619	// depth is multiplied by 16 in case of DSC enable
1620	s32 hblank_symbols;
1621	// number of link clocks per line.
1622	int vblank_symbols	  = 0;
1623	bool bEnableDsc = false;
1624	unsigned surfaceWidth = asyh->mode.h.blanks - asyh->mode.h.blanke;
1625	unsigned rasterWidth = asyh->mode.h.active;
1626	unsigned depth = asyh->or.bpc * 3;
1627	unsigned DSC_FACTOR = bEnableDsc ? 16 : 1;
1628	u64 pixelClockHz = asyh->mode.clock * 1000;
1629	u64 PrecisionFactor = 100000, ratioF, watermarkF;
1630	u32 numLanesPerLink = outp->dp.link_nr;
1631	u32 numSymbolsPerLine;
1632	u32 BlankingBits;
1633	u32 surfaceWidthPerLink;
1634	u32 PixelSteeringBits;
1635	u64 NumBlankingLinkClocks;
1636	u32 MinHBlank;
1637
1638	if (outp->outp.info.dp.increased_wm) {
1639		watermarkAdjust = DP_CONFIG_INCREASED_WATERMARK_ADJUST;
1640		watermarkMinimum = DP_CONFIG_INCREASED_WATERMARK_LIMIT;
1641	}
1642
1643	if ((pixelClockHz * depth) >= (8 * minRate * outp->dp.link_nr * DSC_FACTOR))
1644	{
1645		return false;
1646	}
1647
1648	//
1649	// For DSC, if (pclk * bpp) < (1/64 * orclk * 8 * lanes) then some TU may end up with
1650	// 0 active symbols. This may cause HW hang. Bug 200379426
1651	//
1652	if ((bEnableDsc) &&
1653	    ((pixelClockHz * depth) < div_u64(8 * minRate * outp->dp.link_nr * DSC_FACTOR, 64)))
1654	{
1655		return false;
1656	}
1657
1658	//
1659	//  Perform the SST calculation.
1660	//	For auto mode the watermark calculation does not need to track accumulated error the
1661	//	formulas for manual mode will not work.  So below calculation was extracted from the DTB.
1662	//
1663	ratioF = div_u64((u64)pixelClockHz * depth * PrecisionFactor, DSC_FACTOR);
1664
1665	ratioF = div_u64(ratioF, 8 * (u64) minRate * outp->dp.link_nr);
1666
1667	if (PrecisionFactor < ratioF) // Assert if we will end up with a negative number in below
1668		return false;
1669
1670	watermarkF = div_u64(ratioF * tuSize * (PrecisionFactor - ratioF), PrecisionFactor);
1671	waterMark = (unsigned)(watermarkAdjust + (div_u64(2 * div_u64(depth * PrecisionFactor, 8 * numLanesPerLink * DSC_FACTOR) + watermarkF, PrecisionFactor)));
1672
1673	//
1674	//  Bounds check the watermark
1675	//
1676	numSymbolsPerLine = div_u64(surfaceWidth * depth, 8 * outp->dp.link_nr * DSC_FACTOR);
1677
1678	if (WARN_ON(waterMark > 39 || waterMark > numSymbolsPerLine))
1679		return false;
1680
1681	//
1682	//  Clamp the low side
1683	//
1684	if (waterMark < watermarkMinimum)
1685		waterMark = watermarkMinimum;
1686
1687	//Bits to send BS/BE/Extra symbols due to pixel padding
1688	//Also accounts for enhanced framing.
1689	BlankingBits = 3*8*numLanesPerLink + (enhancedFraming ? 3*8*numLanesPerLink : 0);
1690
1691	//VBID/MVID/MAUD sent 4 times all the time
1692	BlankingBits += 3*8*4;
1693
1694	surfaceWidthPerLink = surfaceWidth;
1695
1696	//Extra bits sent due to pixel steering
1697	u32 remain;
1698	div_u64_rem(surfaceWidthPerLink, numLanesPerLink, &remain);
1699	PixelSteeringBits = remain ? div_u64((numLanesPerLink - remain) * depth, DSC_FACTOR) : 0;
1700
1701	BlankingBits += PixelSteeringBits;
1702	NumBlankingLinkClocks = div_u64((u64)BlankingBits * PrecisionFactor, (8 * numLanesPerLink));
1703	MinHBlank = (u32)(div_u64(div_u64(NumBlankingLinkClocks * pixelClockHz, minRate), PrecisionFactor));
1704	MinHBlank += 12;
1705
1706	if (WARN_ON(MinHBlank > rasterWidth - surfaceWidth))
1707		return false;
1708
1709	// Bug 702290 - Active Width should be greater than 60
1710	if (WARN_ON(surfaceWidth <= 60))
1711		return false;
1712
1713
1714	hblank_symbols = (s32)(div_u64((u64)(rasterWidth - surfaceWidth - MinHBlank) * minRate, pixelClockHz));
1715
1716	//reduce HBlank Symbols to account for secondary data packet
1717	hblank_symbols -= 1; //Stuffer latency to send BS
1718	hblank_symbols -= 3; //SPKT latency to send data to stuffer
1719
1720	hblank_symbols -= numLanesPerLink == 1 ? 9  : numLanesPerLink == 2 ? 6 : 3;
1721
1722	hBlankSym = (hblank_symbols < 0) ? 0 : hblank_symbols;
1723
1724	// Refer to dev_disp.ref for more information.
1725	// # symbols/vblank = ((SetRasterBlankEnd.X + SetRasterSize.Width - SetRasterBlankStart.X - 40) * link_clk / pclk) - Y - 1;
1726	// where Y = (# lanes == 4) 12 : (# lanes == 2) ? 21 : 39
1727	if (surfaceWidth < 40)
1728	{
1729		vblank_symbols = 0;
1730	}
1731	else
1732	{
1733		vblank_symbols = (s32)((div_u64((u64)(surfaceWidth - 40) * minRate, pixelClockHz))) - 1;
1734
1735		vblank_symbols -= numLanesPerLink == 1 ? 39  : numLanesPerLink == 2 ? 21 : 12;
1736	}
1737
1738	vBlankSym = (vblank_symbols < 0) ? 0 : vblank_symbols;
1739
1740	return nvif_outp_dp_sst(&outp->outp, head->base.index, waterMark, hBlankSym, vBlankSym);
1741}
1742
1743static void
1744nv50_sor_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *state)
1745{
1746	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
1747	struct nouveau_crtc *nv_crtc = nv50_outp_get_new_crtc(state, nv_encoder);
1748	struct nv50_head_atom *asyh =
1749		nv50_head_atom(drm_atomic_get_new_crtc_state(state, &nv_crtc->base));
1750	struct drm_display_mode *mode = &asyh->state.adjusted_mode;
1751	struct nv50_disp *disp = nv50_disp(encoder->dev);
1752	struct nv50_head *head = nv50_head(&nv_crtc->base);
1753	struct nvif_outp *outp = &nv_encoder->outp;
1754	struct drm_device *dev = encoder->dev;
1755	struct nouveau_drm *drm = nouveau_drm(dev);
1756	struct nouveau_connector *nv_connector;
1757#ifdef CONFIG_DRM_NOUVEAU_BACKLIGHT
1758	struct nouveau_backlight *backlight;
1759#endif
1760	struct nvbios *bios = &drm->vbios;
1761	bool lvds_dual = false, lvds_8bpc = false, hda = false;
1762	u8 proto = NV507D_SOR_SET_CONTROL_PROTOCOL_CUSTOM;
1763	u8 depth = NV837D_SOR_SET_CONTROL_PIXEL_DEPTH_DEFAULT;
1764
1765	nv_connector = nv50_outp_get_new_connector(state, nv_encoder);
1766	nv_encoder->crtc = &nv_crtc->base;
1767
1768	if ((disp->disp->object.oclass == GT214_DISP ||
1769	     disp->disp->object.oclass >= GF110_DISP) &&
1770	    nv_encoder->dcb->type != DCB_OUTPUT_LVDS &&
1771	    nv_connector->base.display_info.has_audio)
1772		hda = true;
1773
1774	if (!nvif_outp_acquired(outp))
1775		nvif_outp_acquire_sor(outp, hda);
1776
1777	switch (nv_encoder->dcb->type) {
1778	case DCB_OUTPUT_TMDS:
1779		if (disp->disp->object.oclass != NV50_DISP &&
1780		    nv_connector->base.display_info.is_hdmi)
1781			nv50_hdmi_enable(encoder, nv_crtc, nv_connector, state, mode, hda);
1782
1783		if (nv_encoder->outp.or.link & 1) {
1784			proto = NV507D_SOR_SET_CONTROL_PROTOCOL_SINGLE_TMDS_A;
1785			/* Only enable dual-link if:
1786			 *  - Need to (i.e. rate > 165MHz)
1787			 *  - DCB says we can
1788			 *  - Not an HDMI monitor, since there's no dual-link
1789			 *    on HDMI.
1790			 */
1791			if (mode->clock >= 165000 &&
1792			    nv_encoder->dcb->duallink_possible &&
1793			    !nv_connector->base.display_info.is_hdmi)
1794				proto = NV507D_SOR_SET_CONTROL_PROTOCOL_DUAL_TMDS;
1795		} else {
1796			proto = NV507D_SOR_SET_CONTROL_PROTOCOL_SINGLE_TMDS_B;
1797		}
1798		break;
1799	case DCB_OUTPUT_LVDS:
1800		proto = NV507D_SOR_SET_CONTROL_PROTOCOL_LVDS_CUSTOM;
1801
1802		if (bios->fp_no_ddc) {
1803			lvds_dual = bios->fp.dual_link;
1804			lvds_8bpc = bios->fp.if_is_24bit;
1805		} else {
1806			if (nv_connector->type == DCB_CONNECTOR_LVDS_SPWG) {
1807				if (((u8 *)nv_connector->edid)[121] == 2)
1808					lvds_dual = true;
1809			} else
1810			if (mode->clock >= bios->fp.duallink_transition_clk) {
1811				lvds_dual = true;
1812			}
1813
1814			if (lvds_dual) {
1815				if (bios->fp.strapless_is_24bit & 2)
1816					lvds_8bpc = true;
1817			} else {
1818				if (bios->fp.strapless_is_24bit & 1)
1819					lvds_8bpc = true;
1820			}
1821
1822			if (asyh->or.bpc == 8)
1823				lvds_8bpc = true;
1824		}
1825
1826		nvif_outp_lvds(&nv_encoder->outp, lvds_dual, lvds_8bpc);
1827		break;
1828	case DCB_OUTPUT_DP:
1829		nouveau_dp_train(nv_encoder, false, mode->clock, asyh->or.bpc);
1830		nv50_sor_dp_watermark_sst(nv_encoder, head, asyh);
1831		depth = nv50_dp_bpc_to_depth(asyh->or.bpc);
1832
1833		if (nv_encoder->outp.or.link & 1)
1834			proto = NV887D_SOR_SET_CONTROL_PROTOCOL_DP_A;
1835		else
1836			proto = NV887D_SOR_SET_CONTROL_PROTOCOL_DP_B;
1837
1838#ifdef CONFIG_DRM_NOUVEAU_BACKLIGHT
1839		backlight = nv_connector->backlight;
1840		if (backlight && backlight->uses_dpcd)
1841			drm_edp_backlight_enable(&nv_connector->aux, &backlight->edp_info,
1842						 backlight->dev->props.brightness);
1843#endif
1844
1845		break;
1846	default:
1847		BUG();
1848		break;
1849	}
1850
1851	if (head->func->display_id)
1852		head->func->display_id(head, BIT(nv_encoder->outp.id));
1853
1854	nv_encoder->update(nv_encoder, nv_crtc->index, asyh, proto, depth);
1855}
1856
1857static const struct drm_encoder_helper_funcs
1858nv50_sor_help = {
1859	.atomic_check = nv50_outp_atomic_check,
1860	.atomic_enable = nv50_sor_atomic_enable,
1861	.atomic_disable = nv50_sor_atomic_disable,
1862};
1863
1864static void
1865nv50_sor_destroy(struct drm_encoder *encoder)
1866{
1867	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
1868
1869	nv50_mstm_del(&nv_encoder->dp.mstm);
1870	drm_encoder_cleanup(encoder);
1871
1872	if (nv_encoder->dcb->type == DCB_OUTPUT_DP)
1873		mutex_destroy(&nv_encoder->dp.hpd_irq_lock);
1874
1875	nvif_outp_dtor(&nv_encoder->outp);
1876	kfree(encoder);
1877}
1878
1879static const struct drm_encoder_funcs
1880nv50_sor_func = {
1881	.destroy = nv50_sor_destroy,
1882};
1883
1884static int
1885nv50_sor_create(struct nouveau_encoder *nv_encoder)
1886{
1887	struct drm_connector *connector = &nv_encoder->conn->base;
1888	struct nouveau_connector *nv_connector = nouveau_connector(connector);
1889	struct nouveau_drm *drm = nouveau_drm(connector->dev);
1890	struct nvkm_i2c *i2c = nvxx_i2c(drm);
1891	struct drm_encoder *encoder;
1892	struct dcb_output *dcbe = nv_encoder->dcb;
1893	struct nv50_disp *disp = nv50_disp(connector->dev);
1894	int type, ret;
1895
1896	switch (dcbe->type) {
1897	case DCB_OUTPUT_LVDS: type = DRM_MODE_ENCODER_LVDS; break;
1898	case DCB_OUTPUT_TMDS:
1899	case DCB_OUTPUT_DP:
1900	default:
1901		type = DRM_MODE_ENCODER_TMDS;
1902		break;
1903	}
1904
1905	nv_encoder->update = nv50_sor_update;
1906
1907	encoder = to_drm_encoder(nv_encoder);
1908	drm_encoder_init(connector->dev, encoder, &nv50_sor_func, type,
1909			 "sor-%04x-%04x", dcbe->hasht, dcbe->hashm);
1910	drm_encoder_helper_add(encoder, &nv50_sor_help);
1911
1912	drm_connector_attach_encoder(connector, encoder);
1913
1914	disp->core->func->sor->get_caps(disp, nv_encoder, ffs(dcbe->or) - 1);
1915	nv50_outp_dump_caps(drm, nv_encoder);
1916
1917	if (dcbe->type == DCB_OUTPUT_DP) {
1918		mutex_init(&nv_encoder->dp.hpd_irq_lock);
1919
1920		if (disp->disp->object.oclass < GF110_DISP) {
1921			/* HW has no support for address-only
1922			 * transactions, so we're required to
1923			 * use custom I2C-over-AUX code.
1924			 */
1925			struct nvkm_i2c_aux *aux;
1926
1927			aux = nvkm_i2c_aux_find(i2c, dcbe->i2c_index);
1928			if (!aux)
1929				return -EINVAL;
1930
1931			nv_encoder->i2c = &aux->i2c;
1932		} else {
1933			nv_encoder->i2c = &nv_connector->aux.ddc;
1934		}
1935
1936		if (nv_connector->type != DCB_CONNECTOR_eDP && nv_encoder->outp.info.dp.mst) {
1937			ret = nv50_mstm_new(nv_encoder, &nv_connector->aux,
1938					    16, nv_connector->base.base.id,
1939					    &nv_encoder->dp.mstm);
1940			if (ret)
1941				return ret;
1942		}
1943	} else
1944	if (nv_encoder->outp.info.ddc != NVIF_OUTP_DDC_INVALID) {
1945		struct nvkm_i2c_bus *bus =
1946			nvkm_i2c_bus_find(i2c, dcbe->i2c_index);
1947		if (bus)
1948			nv_encoder->i2c = &bus->i2c;
1949	}
1950
1951	return 0;
1952}
1953
1954/******************************************************************************
1955 * PIOR
1956 *****************************************************************************/
1957static int
1958nv50_pior_atomic_check(struct drm_encoder *encoder,
1959		       struct drm_crtc_state *crtc_state,
1960		       struct drm_connector_state *conn_state)
1961{
1962	int ret = nv50_outp_atomic_check(encoder, crtc_state, conn_state);
1963	if (ret)
1964		return ret;
1965	crtc_state->adjusted_mode.clock *= 2;
1966	return 0;
1967}
1968
1969static void
1970nv50_pior_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *state)
1971{
1972	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
1973	struct nv50_core *core = nv50_disp(encoder->dev)->core;
1974	const u32 ctrl = NVDEF(NV507D, PIOR_SET_CONTROL, OWNER, NONE);
1975
1976	core->func->pior->ctrl(core, nv_encoder->outp.or.id, ctrl, NULL);
1977	nv_encoder->crtc = NULL;
1978}
1979
1980static void
1981nv50_pior_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *state)
1982{
1983	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
1984	struct nouveau_crtc *nv_crtc = nv50_outp_get_new_crtc(state, nv_encoder);
1985	struct nv50_head_atom *asyh =
1986		nv50_head_atom(drm_atomic_get_new_crtc_state(state, &nv_crtc->base));
1987	struct nv50_core *core = nv50_disp(encoder->dev)->core;
1988	u32 ctrl = 0;
1989
1990	switch (nv_crtc->index) {
1991	case 0: ctrl |= NVDEF(NV507D, PIOR_SET_CONTROL, OWNER, HEAD0); break;
1992	case 1: ctrl |= NVDEF(NV507D, PIOR_SET_CONTROL, OWNER, HEAD1); break;
1993	default:
1994		WARN_ON(1);
1995		break;
1996	}
1997
1998	switch (asyh->or.bpc) {
1999	case 10: asyh->or.depth = NV837D_PIOR_SET_CONTROL_PIXEL_DEPTH_BPP_30_444; break;
2000	case  8: asyh->or.depth = NV837D_PIOR_SET_CONTROL_PIXEL_DEPTH_BPP_24_444; break;
2001	case  6: asyh->or.depth = NV837D_PIOR_SET_CONTROL_PIXEL_DEPTH_BPP_18_444; break;
2002	default: asyh->or.depth = NV837D_PIOR_SET_CONTROL_PIXEL_DEPTH_DEFAULT; break;
2003	}
2004
2005	if (!nvif_outp_acquired(&nv_encoder->outp))
2006		nvif_outp_acquire_pior(&nv_encoder->outp);
2007
2008	switch (nv_encoder->dcb->type) {
2009	case DCB_OUTPUT_TMDS:
2010		ctrl |= NVDEF(NV507D, PIOR_SET_CONTROL, PROTOCOL, EXT_TMDS_ENC);
2011		break;
2012	case DCB_OUTPUT_DP:
2013		ctrl |= NVDEF(NV507D, PIOR_SET_CONTROL, PROTOCOL, EXT_TMDS_ENC);
2014		nouveau_dp_train(nv_encoder, false, asyh->state.adjusted_mode.clock, 6);
2015		break;
2016	default:
2017		BUG();
2018		break;
2019	}
2020
2021	core->func->pior->ctrl(core, nv_encoder->outp.or.id, ctrl, asyh);
2022	nv_encoder->crtc = &nv_crtc->base;
2023}
2024
2025static const struct drm_encoder_helper_funcs
2026nv50_pior_help = {
2027	.atomic_check = nv50_pior_atomic_check,
2028	.atomic_enable = nv50_pior_atomic_enable,
2029	.atomic_disable = nv50_pior_atomic_disable,
2030};
2031
2032static void
2033nv50_pior_destroy(struct drm_encoder *encoder)
2034{
2035	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
2036
2037	nvif_outp_dtor(&nv_encoder->outp);
2038
2039	drm_encoder_cleanup(encoder);
2040
2041	mutex_destroy(&nv_encoder->dp.hpd_irq_lock);
2042	kfree(encoder);
2043}
2044
2045static const struct drm_encoder_funcs
2046nv50_pior_func = {
2047	.destroy = nv50_pior_destroy,
2048};
2049
2050static int
2051nv50_pior_create(struct nouveau_encoder *nv_encoder)
2052{
2053	struct drm_connector *connector = &nv_encoder->conn->base;
2054	struct drm_device *dev = connector->dev;
2055	struct nouveau_drm *drm = nouveau_drm(dev);
2056	struct nv50_disp *disp = nv50_disp(dev);
2057	struct nvkm_i2c *i2c = nvxx_i2c(drm);
2058	struct nvkm_i2c_bus *bus = NULL;
2059	struct nvkm_i2c_aux *aux = NULL;
2060	struct i2c_adapter *ddc;
2061	struct drm_encoder *encoder;
2062	struct dcb_output *dcbe = nv_encoder->dcb;
2063	int type;
2064
2065	switch (dcbe->type) {
2066	case DCB_OUTPUT_TMDS:
2067		bus  = nvkm_i2c_bus_find(i2c, nv_encoder->outp.info.ddc);
2068		ddc  = bus ? &bus->i2c : NULL;
2069		type = DRM_MODE_ENCODER_TMDS;
2070		break;
2071	case DCB_OUTPUT_DP:
2072		aux  = nvkm_i2c_aux_find(i2c, nv_encoder->outp.info.dp.aux);
2073		ddc  = aux ? &aux->i2c : NULL;
2074		type = DRM_MODE_ENCODER_TMDS;
2075		break;
2076	default:
2077		return -ENODEV;
2078	}
2079
2080	nv_encoder->i2c = ddc;
2081
2082	mutex_init(&nv_encoder->dp.hpd_irq_lock);
2083
2084	encoder = to_drm_encoder(nv_encoder);
2085	drm_encoder_init(connector->dev, encoder, &nv50_pior_func, type,
2086			 "pior-%04x-%04x", dcbe->hasht, dcbe->hashm);
2087	drm_encoder_helper_add(encoder, &nv50_pior_help);
2088
2089	drm_connector_attach_encoder(connector, encoder);
2090
2091	disp->core->func->pior->get_caps(disp, nv_encoder, ffs(dcbe->or) - 1);
2092	nv50_outp_dump_caps(drm, nv_encoder);
2093
2094	return 0;
2095}
2096
2097/******************************************************************************
2098 * Atomic
2099 *****************************************************************************/
2100
2101static void
2102nv50_disp_atomic_commit_core(struct drm_atomic_state *state, u32 *interlock)
2103{
2104	struct drm_dp_mst_topology_mgr *mgr;
2105	struct drm_dp_mst_topology_state *mst_state;
2106	struct nouveau_drm *drm = nouveau_drm(state->dev);
2107	struct nv50_disp *disp = nv50_disp(drm->dev);
2108	struct nv50_atom *atom = nv50_atom(state);
2109	struct nv50_core *core = disp->core;
2110	struct nv50_outp_atom *outp;
2111	struct nv50_mstm *mstm;
2112	int i;
2113
2114	NV_ATOMIC(drm, "commit core %08x\n", interlock[NV50_DISP_INTERLOCK_BASE]);
2115
2116	for_each_new_mst_mgr_in_state(state, mgr, mst_state, i) {
2117		mstm = nv50_mstm(mgr);
2118		if (mstm->modified)
2119			nv50_mstm_prepare(state, mst_state, mstm);
2120	}
2121
2122	core->func->ntfy_init(disp->sync, NV50_DISP_CORE_NTFY);
2123	core->func->update(core, interlock, true);
2124	if (core->func->ntfy_wait_done(disp->sync, NV50_DISP_CORE_NTFY,
2125				       disp->core->chan.base.device))
2126		NV_ERROR(drm, "core notifier timeout\n");
2127
2128	for_each_new_mst_mgr_in_state(state, mgr, mst_state, i) {
2129		mstm = nv50_mstm(mgr);
2130		if (mstm->modified)
2131			nv50_mstm_cleanup(state, mst_state, mstm);
2132	}
2133
2134	list_for_each_entry(outp, &atom->outp, head) {
2135		if (outp->encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
2136			struct nouveau_encoder *nv_encoder = nouveau_encoder(outp->encoder);
2137
2138			if (outp->enabled) {
2139				nv50_audio_enable(outp->encoder, nouveau_crtc(nv_encoder->crtc),
2140						  nv_encoder->conn, NULL, NULL);
2141				outp->enabled = outp->disabled = false;
2142			} else {
2143				if (outp->disabled) {
2144					nvif_outp_release(&nv_encoder->outp);
2145					outp->disabled = false;
2146				}
2147			}
2148		}
2149	}
2150}
2151
2152static void
2153nv50_disp_atomic_commit_wndw(struct drm_atomic_state *state, u32 *interlock)
2154{
2155	struct drm_plane_state *new_plane_state;
2156	struct drm_plane *plane;
2157	int i;
2158
2159	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
2160		struct nv50_wndw *wndw = nv50_wndw(plane);
2161		if (interlock[wndw->interlock.type] & wndw->interlock.data) {
2162			if (wndw->func->update)
2163				wndw->func->update(wndw, interlock);
2164		}
2165	}
2166}
2167
2168static void
2169nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
2170{
2171	struct drm_device *dev = state->dev;
2172	struct drm_crtc_state *new_crtc_state, *old_crtc_state;
2173	struct drm_crtc *crtc;
2174	struct drm_plane_state *new_plane_state;
2175	struct drm_plane *plane;
2176	struct nouveau_drm *drm = nouveau_drm(dev);
2177	struct nv50_disp *disp = nv50_disp(dev);
2178	struct nv50_atom *atom = nv50_atom(state);
2179	struct nv50_core *core = disp->core;
2180	struct nv50_outp_atom *outp, *outt;
2181	u32 interlock[NV50_DISP_INTERLOCK__SIZE] = {};
2182	int i;
2183	bool flushed = false;
2184
2185	NV_ATOMIC(drm, "commit %d %d\n", atom->lock_core, atom->flush_disable);
2186	nv50_crc_atomic_stop_reporting(state);
2187	drm_atomic_helper_wait_for_fences(dev, state, false);
2188	drm_atomic_helper_wait_for_dependencies(state);
2189	drm_dp_mst_atomic_wait_for_dependencies(state);
2190	drm_atomic_helper_update_legacy_modeset_state(dev, state);
2191	drm_atomic_helper_calc_timestamping_constants(state);
2192
2193	if (atom->lock_core)
2194		mutex_lock(&disp->mutex);
2195
2196	/* Disable head(s). */
2197	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
2198		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
2199		struct nv50_head *head = nv50_head(crtc);
2200
2201		NV_ATOMIC(drm, "%s: clr %04x (set %04x)\n", crtc->name,
2202			  asyh->clr.mask, asyh->set.mask);
2203
2204		if (old_crtc_state->active && !new_crtc_state->active) {
2205			pm_runtime_put_noidle(dev->dev);
2206			drm_crtc_vblank_off(crtc);
2207		}
2208
2209		if (asyh->clr.mask) {
2210			nv50_head_flush_clr(head, asyh, atom->flush_disable);
2211			interlock[NV50_DISP_INTERLOCK_CORE] |= 1;
2212		}
2213	}
2214
2215	/* Disable plane(s). */
2216	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
2217		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
2218		struct nv50_wndw *wndw = nv50_wndw(plane);
2219
2220		NV_ATOMIC(drm, "%s: clr %02x (set %02x)\n", plane->name,
2221			  asyw->clr.mask, asyw->set.mask);
2222		if (!asyw->clr.mask)
2223			continue;
2224
2225		nv50_wndw_flush_clr(wndw, interlock, atom->flush_disable, asyw);
2226	}
2227
2228	/* Disable output path(s). */
2229	list_for_each_entry(outp, &atom->outp, head) {
2230		const struct drm_encoder_helper_funcs *help;
2231		struct drm_encoder *encoder;
2232
2233		encoder = outp->encoder;
2234		help = encoder->helper_private;
2235
2236		NV_ATOMIC(drm, "%s: clr %02x (set %02x)\n", encoder->name,
2237			  outp->clr.mask, outp->set.mask);
2238
2239		if (outp->clr.mask) {
2240			help->atomic_disable(encoder, state);
2241			outp->disabled = true;
2242			interlock[NV50_DISP_INTERLOCK_CORE] |= 1;
2243		}
2244	}
2245
2246	/* Flush disable. */
2247	if (interlock[NV50_DISP_INTERLOCK_CORE]) {
2248		if (atom->flush_disable) {
2249			nv50_disp_atomic_commit_wndw(state, interlock);
2250			nv50_disp_atomic_commit_core(state, interlock);
2251			memset(interlock, 0x00, sizeof(interlock));
2252
2253			flushed = true;
2254		}
2255	}
2256
2257	if (flushed)
2258		nv50_crc_atomic_release_notifier_contexts(state);
2259	nv50_crc_atomic_init_notifier_contexts(state);
2260
2261	/* Update output path(s). */
2262	list_for_each_entry(outp, &atom->outp, head) {
2263		const struct drm_encoder_helper_funcs *help;
2264		struct drm_encoder *encoder;
2265
2266		encoder = outp->encoder;
2267		help = encoder->helper_private;
2268
2269		NV_ATOMIC(drm, "%s: set %02x (clr %02x)\n", encoder->name,
2270			  outp->set.mask, outp->clr.mask);
2271
2272		if (outp->set.mask) {
2273			help->atomic_enable(encoder, state);
2274			outp->enabled = true;
2275			interlock[NV50_DISP_INTERLOCK_CORE] = 1;
2276		}
2277	}
2278
2279	/* Update head(s). */
2280	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
2281		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
2282		struct nv50_head *head = nv50_head(crtc);
2283
2284		NV_ATOMIC(drm, "%s: set %04x (clr %04x)\n", crtc->name,
2285			  asyh->set.mask, asyh->clr.mask);
2286
2287		if (asyh->set.mask) {
2288			nv50_head_flush_set(head, asyh);
2289			interlock[NV50_DISP_INTERLOCK_CORE] = 1;
2290		}
2291
2292		if (new_crtc_state->active) {
2293			if (!old_crtc_state->active) {
2294				drm_crtc_vblank_on(crtc);
2295				pm_runtime_get_noresume(dev->dev);
2296			}
2297			if (new_crtc_state->event)
2298				drm_crtc_vblank_get(crtc);
2299		}
2300	}
2301
2302	/* Update window->head assignment.
2303	 *
2304	 * This has to happen in an update that's not interlocked with
2305	 * any window channels to avoid hitting HW error checks.
2306	 *
2307	 *TODO: Proper handling of window ownership (Turing apparently
2308	 *      supports non-fixed mappings).
2309	 */
2310	if (core->assign_windows) {
2311		core->func->wndw.owner(core);
2312		nv50_disp_atomic_commit_core(state, interlock);
2313		core->assign_windows = false;
2314		interlock[NV50_DISP_INTERLOCK_CORE] = 0;
2315	}
2316
2317	/* Finish updating head(s)...
2318	 *
2319	 * NVD is rather picky about both where window assignments can change,
2320	 * *and* about certain core and window channel states matching.
2321	 *
2322	 * The EFI GOP driver on newer GPUs configures window channels with a
2323	 * different output format to what we do, and the core channel update
2324	 * in the assign_windows case above would result in a state mismatch.
2325	 *
2326	 * Delay some of the head update until after that point to workaround
2327	 * the issue.  This only affects the initial modeset.
2328	 *
2329	 * TODO: handle this better when adding flexible window mapping
2330	 */
2331	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
2332		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
2333		struct nv50_head *head = nv50_head(crtc);
2334
2335		NV_ATOMIC(drm, "%s: set %04x (clr %04x)\n", crtc->name,
2336			  asyh->set.mask, asyh->clr.mask);
2337
2338		if (asyh->set.mask) {
2339			nv50_head_flush_set_wndw(head, asyh);
2340			interlock[NV50_DISP_INTERLOCK_CORE] = 1;
2341		}
2342	}
2343
2344	/* Update plane(s). */
2345	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
2346		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
2347		struct nv50_wndw *wndw = nv50_wndw(plane);
2348
2349		NV_ATOMIC(drm, "%s: set %02x (clr %02x)\n", plane->name,
2350			  asyw->set.mask, asyw->clr.mask);
2351		if ( !asyw->set.mask &&
2352		    (!asyw->clr.mask || atom->flush_disable))
2353			continue;
2354
2355		nv50_wndw_flush_set(wndw, interlock, asyw);
2356	}
2357
2358	/* Flush update. */
2359	nv50_disp_atomic_commit_wndw(state, interlock);
2360
2361	if (interlock[NV50_DISP_INTERLOCK_CORE]) {
2362		if (interlock[NV50_DISP_INTERLOCK_BASE] ||
2363		    interlock[NV50_DISP_INTERLOCK_OVLY] ||
2364		    interlock[NV50_DISP_INTERLOCK_WNDW] ||
2365		    !atom->state.legacy_cursor_update)
2366			nv50_disp_atomic_commit_core(state, interlock);
2367		else
2368			disp->core->func->update(disp->core, interlock, false);
2369	}
2370
2371	if (atom->lock_core)
2372		mutex_unlock(&disp->mutex);
2373
2374	list_for_each_entry_safe(outp, outt, &atom->outp, head) {
2375		list_del(&outp->head);
2376		kfree(outp);
2377	}
2378
2379	/* Wait for HW to signal completion. */
2380	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
2381		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
2382		struct nv50_wndw *wndw = nv50_wndw(plane);
2383		int ret = nv50_wndw_wait_armed(wndw, asyw);
2384		if (ret)
2385			NV_ERROR(drm, "%s: timeout\n", plane->name);
2386	}
2387
2388	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
2389		if (new_crtc_state->event) {
2390			unsigned long flags;
2391			/* Get correct count/ts if racing with vblank irq */
2392			if (new_crtc_state->active)
2393				drm_crtc_accurate_vblank_count(crtc);
2394			spin_lock_irqsave(&crtc->dev->event_lock, flags);
2395			drm_crtc_send_vblank_event(crtc, new_crtc_state->event);
2396			spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
2397
2398			new_crtc_state->event = NULL;
2399			if (new_crtc_state->active)
2400				drm_crtc_vblank_put(crtc);
2401		}
2402	}
2403
2404	nv50_crc_atomic_start_reporting(state);
2405	if (!flushed)
2406		nv50_crc_atomic_release_notifier_contexts(state);
2407
2408	drm_atomic_helper_commit_hw_done(state);
2409	drm_atomic_helper_cleanup_planes(dev, state);
2410	drm_atomic_helper_commit_cleanup_done(state);
2411	drm_atomic_state_put(state);
2412
2413	/* Drop the RPM ref we got from nv50_disp_atomic_commit() */
2414	pm_runtime_mark_last_busy(dev->dev);
2415	pm_runtime_put_autosuspend(dev->dev);
2416}
2417
2418static void
2419nv50_disp_atomic_commit_work(struct work_struct *work)
2420{
2421	struct drm_atomic_state *state =
2422		container_of(work, typeof(*state), commit_work);
2423	nv50_disp_atomic_commit_tail(state);
2424}
2425
2426static int
2427nv50_disp_atomic_commit(struct drm_device *dev,
2428			struct drm_atomic_state *state, bool nonblock)
2429{
2430	struct drm_plane_state *new_plane_state;
2431	struct drm_plane *plane;
2432	int ret, i;
2433
2434	ret = pm_runtime_get_sync(dev->dev);
2435	if (ret < 0 && ret != -EACCES) {
2436		pm_runtime_put_autosuspend(dev->dev);
2437		return ret;
2438	}
2439
2440	ret = drm_atomic_helper_setup_commit(state, nonblock);
2441	if (ret)
2442		goto done;
2443
2444	INIT_WORK(&state->commit_work, nv50_disp_atomic_commit_work);
2445
2446	ret = drm_atomic_helper_prepare_planes(dev, state);
2447	if (ret)
2448		goto done;
2449
2450	if (!nonblock) {
2451		ret = drm_atomic_helper_wait_for_fences(dev, state, true);
2452		if (ret)
2453			goto err_cleanup;
2454	}
2455
2456	ret = drm_atomic_helper_swap_state(state, true);
2457	if (ret)
2458		goto err_cleanup;
2459
2460	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
2461		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
2462		struct nv50_wndw *wndw = nv50_wndw(plane);
2463
2464		if (asyw->set.image)
2465			nv50_wndw_ntfy_enable(wndw, asyw);
2466	}
2467
2468	drm_atomic_state_get(state);
2469
2470	/*
2471	 * Grab another RPM ref for the commit tail, which will release the
2472	 * ref when it's finished
2473	 */
2474	pm_runtime_get_noresume(dev->dev);
2475
2476	if (nonblock)
2477		queue_work(system_unbound_wq, &state->commit_work);
2478	else
2479		nv50_disp_atomic_commit_tail(state);
2480
2481err_cleanup:
2482	if (ret)
2483		drm_atomic_helper_unprepare_planes(dev, state);
2484done:
2485	pm_runtime_put_autosuspend(dev->dev);
2486	return ret;
2487}
2488
2489static struct nv50_outp_atom *
2490nv50_disp_outp_atomic_add(struct nv50_atom *atom, struct drm_encoder *encoder)
2491{
2492	struct nv50_outp_atom *outp;
2493
2494	list_for_each_entry(outp, &atom->outp, head) {
2495		if (outp->encoder == encoder)
2496			return outp;
2497	}
2498
2499	outp = kzalloc(sizeof(*outp), GFP_KERNEL);
2500	if (!outp)
2501		return ERR_PTR(-ENOMEM);
2502
2503	list_add(&outp->head, &atom->outp);
2504	outp->encoder = encoder;
2505	return outp;
2506}
2507
2508static int
2509nv50_disp_outp_atomic_check_clr(struct nv50_atom *atom,
2510				struct drm_connector_state *old_connector_state)
2511{
2512	struct drm_encoder *encoder = old_connector_state->best_encoder;
2513	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
2514	struct drm_crtc *crtc;
2515	struct nv50_outp_atom *outp;
2516
2517	if (!(crtc = old_connector_state->crtc))
2518		return 0;
2519
2520	old_crtc_state = drm_atomic_get_old_crtc_state(&atom->state, crtc);
2521	new_crtc_state = drm_atomic_get_new_crtc_state(&atom->state, crtc);
2522	if (old_crtc_state->active && drm_atomic_crtc_needs_modeset(new_crtc_state)) {
2523		outp = nv50_disp_outp_atomic_add(atom, encoder);
2524		if (IS_ERR(outp))
2525			return PTR_ERR(outp);
2526
2527		if (outp->encoder->encoder_type == DRM_MODE_ENCODER_DPMST ||
2528		    nouveau_encoder(outp->encoder)->dcb->type == DCB_OUTPUT_DP)
2529			atom->flush_disable = true;
2530		outp->clr.ctrl = true;
2531		atom->lock_core = true;
2532	}
2533
2534	return 0;
2535}
2536
2537static int
2538nv50_disp_outp_atomic_check_set(struct nv50_atom *atom,
2539				struct drm_connector_state *connector_state)
2540{
2541	struct drm_encoder *encoder = connector_state->best_encoder;
2542	struct drm_crtc_state *new_crtc_state;
2543	struct drm_crtc *crtc;
2544	struct nv50_outp_atom *outp;
2545
2546	if (!(crtc = connector_state->crtc))
2547		return 0;
2548
2549	new_crtc_state = drm_atomic_get_new_crtc_state(&atom->state, crtc);
2550	if (new_crtc_state->active && drm_atomic_crtc_needs_modeset(new_crtc_state)) {
2551		outp = nv50_disp_outp_atomic_add(atom, encoder);
2552		if (IS_ERR(outp))
2553			return PTR_ERR(outp);
2554
2555		outp->set.ctrl = true;
2556		atom->lock_core = true;
2557	}
2558
2559	return 0;
2560}
2561
2562static int
2563nv50_disp_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
2564{
2565	struct nv50_atom *atom = nv50_atom(state);
2566	struct nv50_core *core = nv50_disp(dev)->core;
2567	struct drm_connector_state *old_connector_state, *new_connector_state;
2568	struct drm_connector *connector;
2569	struct drm_crtc_state *new_crtc_state;
2570	struct drm_crtc *crtc;
2571	struct nv50_head *head;
2572	struct nv50_head_atom *asyh;
2573	int ret, i;
2574
2575	if (core->assign_windows && core->func->head->static_wndw_map) {
2576		drm_for_each_crtc(crtc, dev) {
2577			new_crtc_state = drm_atomic_get_crtc_state(state,
2578								   crtc);
2579			if (IS_ERR(new_crtc_state))
2580				return PTR_ERR(new_crtc_state);
2581
2582			head = nv50_head(crtc);
2583			asyh = nv50_head_atom(new_crtc_state);
2584			core->func->head->static_wndw_map(head, asyh);
2585		}
2586	}
2587
2588	/* We need to handle colour management on a per-plane basis. */
2589	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
2590		if (new_crtc_state->color_mgmt_changed) {
2591			ret = drm_atomic_add_affected_planes(state, crtc);
2592			if (ret)
2593				return ret;
2594		}
2595	}
2596
2597	ret = drm_atomic_helper_check(dev, state);
2598	if (ret)
2599		return ret;
2600
2601	for_each_oldnew_connector_in_state(state, connector, old_connector_state, new_connector_state, i) {
2602		ret = nv50_disp_outp_atomic_check_clr(atom, old_connector_state);
2603		if (ret)
2604			return ret;
2605
2606		ret = nv50_disp_outp_atomic_check_set(atom, new_connector_state);
2607		if (ret)
2608			return ret;
2609	}
2610
2611	ret = drm_dp_mst_atomic_check(state);
2612	if (ret)
2613		return ret;
2614
2615	nv50_crc_atomic_check_outp(atom);
2616
2617	return 0;
2618}
2619
2620static void
2621nv50_disp_atomic_state_clear(struct drm_atomic_state *state)
2622{
2623	struct nv50_atom *atom = nv50_atom(state);
2624	struct nv50_outp_atom *outp, *outt;
2625
2626	list_for_each_entry_safe(outp, outt, &atom->outp, head) {
2627		list_del(&outp->head);
2628		kfree(outp);
2629	}
2630
2631	drm_atomic_state_default_clear(state);
2632}
2633
2634static void
2635nv50_disp_atomic_state_free(struct drm_atomic_state *state)
2636{
2637	struct nv50_atom *atom = nv50_atom(state);
2638	drm_atomic_state_default_release(&atom->state);
2639	kfree(atom);
2640}
2641
2642static struct drm_atomic_state *
2643nv50_disp_atomic_state_alloc(struct drm_device *dev)
2644{
2645	struct nv50_atom *atom;
2646	if (!(atom = kzalloc(sizeof(*atom), GFP_KERNEL)) ||
2647	    drm_atomic_state_init(dev, &atom->state) < 0) {
2648		kfree(atom);
2649		return NULL;
2650	}
2651	INIT_LIST_HEAD(&atom->outp);
2652	return &atom->state;
2653}
2654
2655static const struct drm_mode_config_funcs
2656nv50_disp_func = {
2657	.fb_create = nouveau_user_framebuffer_create,
2658	.atomic_check = nv50_disp_atomic_check,
2659	.atomic_commit = nv50_disp_atomic_commit,
2660	.atomic_state_alloc = nv50_disp_atomic_state_alloc,
2661	.atomic_state_clear = nv50_disp_atomic_state_clear,
2662	.atomic_state_free = nv50_disp_atomic_state_free,
2663};
2664
2665static const struct drm_mode_config_helper_funcs
2666nv50_disp_helper_func = {
2667	.atomic_commit_setup = drm_dp_mst_atomic_setup_commit,
2668};
2669
2670/******************************************************************************
2671 * Init
2672 *****************************************************************************/
2673
2674static void
2675nv50_display_fini(struct drm_device *dev, bool runtime, bool suspend)
2676{
2677	struct nouveau_drm *drm = nouveau_drm(dev);
2678	struct drm_encoder *encoder;
2679
2680	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
2681		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST)
2682			nv50_mstm_fini(nouveau_encoder(encoder));
2683	}
2684
2685	if (!runtime && !drm->headless)
2686		cancel_work_sync(&drm->hpd_work);
2687}
2688
2689static inline void
2690nv50_display_read_hw_or_state(struct drm_device *dev, struct nv50_disp *disp,
2691			      struct nouveau_encoder *outp)
2692{
2693	struct drm_crtc *crtc;
2694	struct drm_connector_list_iter conn_iter;
2695	struct drm_connector *conn;
2696	struct nv50_head_atom *armh;
2697	const u32 encoder_mask = drm_encoder_mask(&outp->base.base);
2698	bool found_conn = false, found_head = false;
2699	u8 proto;
2700	int head_idx;
2701	int ret;
2702
2703	switch (outp->dcb->type) {
2704	case DCB_OUTPUT_TMDS:
2705		ret = nvif_outp_inherit_tmds(&outp->outp, &proto);
2706		break;
2707	case DCB_OUTPUT_DP:
2708		ret = nvif_outp_inherit_dp(&outp->outp, &proto);
2709		break;
2710	case DCB_OUTPUT_LVDS:
2711		ret = nvif_outp_inherit_lvds(&outp->outp, &proto);
2712		break;
2713	case DCB_OUTPUT_ANALOG:
2714		ret = nvif_outp_inherit_rgb_crt(&outp->outp, &proto);
2715		break;
2716	default:
2717		drm_dbg_kms(dev, "Readback for %s not implemented yet, skipping\n",
2718			    outp->base.base.name);
2719		drm_WARN_ON(dev, true);
2720		return;
2721	}
2722
2723	if (ret < 0)
2724		return;
2725
2726	head_idx = ret;
2727
2728	drm_for_each_crtc(crtc, dev) {
2729		if (crtc->index != head_idx)
2730			continue;
2731
2732		armh = nv50_head_atom(crtc->state);
2733		found_head = true;
2734		break;
2735	}
2736	if (drm_WARN_ON(dev, !found_head))
2737		return;
2738
2739	/* Figure out which connector is being used by this encoder */
2740	drm_connector_list_iter_begin(dev, &conn_iter);
2741	nouveau_for_each_non_mst_connector_iter(conn, &conn_iter) {
2742		if (nouveau_connector(conn)->index == outp->dcb->connector) {
2743			found_conn = true;
2744			break;
2745		}
2746	}
2747	drm_connector_list_iter_end(&conn_iter);
2748	if (drm_WARN_ON(dev, !found_conn))
2749		return;
2750
2751	armh->state.encoder_mask = encoder_mask;
2752	armh->state.connector_mask = drm_connector_mask(conn);
2753	armh->state.active = true;
2754	armh->state.enable = true;
2755	pm_runtime_get_noresume(dev->dev);
2756
2757	outp->crtc = crtc;
2758	outp->ctrl = NVVAL(NV507D, SOR_SET_CONTROL, PROTOCOL, proto) | BIT(crtc->index);
2759
2760	drm_connector_get(conn);
2761	conn->state->crtc = crtc;
2762	conn->state->best_encoder = &outp->base.base;
2763}
2764
2765/* Read back the currently programmed display state */
2766static void
2767nv50_display_read_hw_state(struct nouveau_drm *drm)
2768{
2769	struct drm_device *dev = drm->dev;
2770	struct drm_encoder *encoder;
2771	struct drm_modeset_acquire_ctx ctx;
2772	struct nv50_disp *disp = nv50_disp(dev);
2773	int ret;
2774
2775	DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, 0, ret);
2776
2777	drm_for_each_encoder(encoder, dev) {
2778		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST)
2779			continue;
2780
2781		nv50_display_read_hw_or_state(dev, disp, nouveau_encoder(encoder));
2782	}
2783
2784	DRM_MODESET_LOCK_ALL_END(dev, ctx, ret);
2785}
2786
2787static int
2788nv50_display_init(struct drm_device *dev, bool resume, bool runtime)
2789{
2790	struct nv50_core *core = nv50_disp(dev)->core;
2791	struct drm_encoder *encoder;
2792
2793	if (resume || runtime)
2794		core->func->init(core);
2795
2796	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
2797		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
2798			struct nouveau_encoder *nv_encoder =
2799				nouveau_encoder(encoder);
2800			nv50_mstm_init(nv_encoder, runtime);
2801		}
2802	}
2803
2804	if (!resume)
2805		nv50_display_read_hw_state(nouveau_drm(dev));
2806
2807	return 0;
2808}
2809
2810static void
2811nv50_display_destroy(struct drm_device *dev)
2812{
2813	struct nv50_disp *disp = nv50_disp(dev);
2814
2815	nv50_audio_component_fini(nouveau_drm(dev));
2816
2817	nvif_object_unmap(&disp->caps);
2818	nvif_object_dtor(&disp->caps);
2819	nv50_core_del(&disp->core);
2820
2821	nouveau_bo_unpin_del(&disp->sync);
2822
2823	nouveau_display(dev)->priv = NULL;
2824	kfree(disp);
2825}
2826
2827int
2828nv50_display_create(struct drm_device *dev)
2829{
2830	struct nouveau_drm *drm = nouveau_drm(dev);
2831	struct drm_connector *connector, *tmp;
2832	struct nv50_disp *disp;
2833	int ret, i;
2834	bool has_mst = false;
2835
2836	disp = kzalloc(sizeof(*disp), GFP_KERNEL);
2837	if (!disp)
2838		return -ENOMEM;
2839
2840	mutex_init(&disp->mutex);
2841
2842	nouveau_display(dev)->priv = disp;
2843	nouveau_display(dev)->dtor = nv50_display_destroy;
2844	nouveau_display(dev)->init = nv50_display_init;
2845	nouveau_display(dev)->fini = nv50_display_fini;
2846	disp->disp = &nouveau_display(dev)->disp;
2847	dev->mode_config.funcs = &nv50_disp_func;
2848	dev->mode_config.helper_private = &nv50_disp_helper_func;
2849	dev->mode_config.quirk_addfb_prefer_xbgr_30bpp = true;
2850	dev->mode_config.normalize_zpos = true;
2851
2852	/* small shared memory area we use for notifiers and semaphores */
2853	ret = nouveau_bo_new_map(&drm->client, NOUVEAU_GEM_DOMAIN_VRAM, PAGE_SIZE, &disp->sync);
2854	if (ret)
2855		goto out;
2856
2857	/* allocate master evo channel */
2858	ret = nv50_core_new(drm, &disp->core);
2859	if (ret)
2860		goto out;
2861
2862	disp->core->func->init(disp->core);
2863	if (disp->core->func->caps_init) {
2864		ret = disp->core->func->caps_init(drm, disp);
2865		if (ret)
2866			goto out;
2867	}
2868
2869	/* Assign the correct format modifiers */
2870	if (disp->disp->object.oclass >= GB202_DISP)
2871		nouveau_display(dev)->format_modifiers = wndwca7e_modifiers;
2872	else if (disp->disp->object.oclass >= TU102_DISP)
2873		nouveau_display(dev)->format_modifiers = wndwc57e_modifiers;
2874	else
2875	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_FERMI)
2876		nouveau_display(dev)->format_modifiers = disp90xx_modifiers;
2877	else
2878		nouveau_display(dev)->format_modifiers = disp50xx_modifiers;
2879
2880	/* FIXME: 256x256 cursors are supported on Kepler, however unlike Maxwell and later
2881	 * generations Kepler requires that we use small pages (4K) for cursor scanout surfaces. The
2882	 * proper fix for this is to teach nouveau to migrate fbs being used for the cursor plane to
2883	 * small page allocations in prepare_fb(). When this is implemented, we should also force
2884	 * large pages (128K) for ovly fbs in order to fix Kepler ovlys.
2885	 * But until then, just limit cursors to 128x128 - which is small enough to avoid ever using
2886	 * large pages.
2887	 */
2888	if (disp->disp->object.oclass >= GM107_DISP) {
2889		dev->mode_config.cursor_width = 256;
2890		dev->mode_config.cursor_height = 256;
2891	} else if (disp->disp->object.oclass >= GK104_DISP) {
2892		dev->mode_config.cursor_width = 128;
2893		dev->mode_config.cursor_height = 128;
2894	} else {
2895		dev->mode_config.cursor_width = 64;
2896		dev->mode_config.cursor_height = 64;
2897	}
2898
2899	/* create encoder/connector objects based on VBIOS DCB table */
2900	for_each_set_bit(i, &disp->disp->outp_mask, sizeof(disp->disp->outp_mask) * 8) {
2901		struct nouveau_encoder *outp;
2902
2903		outp = kzalloc(sizeof(*outp), GFP_KERNEL);
2904		if (!outp)
2905			break;
2906
2907		ret = nvif_outp_ctor(disp->disp, "kmsOutp", i, &outp->outp);
2908		if (ret) {
2909			kfree(outp);
2910			continue;
2911		}
2912
2913		connector = nouveau_connector_create(dev, outp->outp.info.conn);
2914		if (IS_ERR(connector)) {
2915			nvif_outp_dtor(&outp->outp);
2916			kfree(outp);
2917			continue;
2918		}
2919
2920		outp->base.base.possible_crtcs = outp->outp.info.heads;
2921		outp->base.base.possible_clones = 0;
2922		outp->conn = nouveau_connector(connector);
2923
2924		outp->dcb = kzalloc(sizeof(*outp->dcb), GFP_KERNEL);
2925		if (!outp->dcb)
2926			break;
2927
2928		switch (outp->outp.info.proto) {
2929		case NVIF_OUTP_RGB_CRT:
2930			outp->dcb->type = DCB_OUTPUT_ANALOG;
2931			outp->dcb->crtconf.maxfreq = outp->outp.info.rgb_crt.freq_max;
2932			break;
2933		case NVIF_OUTP_TMDS:
2934			outp->dcb->type = DCB_OUTPUT_TMDS;
2935			outp->dcb->duallink_possible = outp->outp.info.tmds.dual;
2936			break;
2937		case NVIF_OUTP_LVDS:
2938			outp->dcb->type = DCB_OUTPUT_LVDS;
2939			outp->dcb->lvdsconf.use_acpi_for_edid = outp->outp.info.lvds.acpi_edid;
2940			break;
2941		case NVIF_OUTP_DP:
2942			outp->dcb->type = DCB_OUTPUT_DP;
2943			outp->dcb->dpconf.link_nr = outp->outp.info.dp.link_nr;
2944			outp->dcb->dpconf.link_bw = outp->outp.info.dp.link_bw;
2945			if (outp->outp.info.dp.mst)
2946				has_mst = true;
2947			break;
2948		default:
2949			WARN_ON(1);
2950			continue;
2951		}
2952
2953		outp->dcb->heads = outp->outp.info.heads;
2954		outp->dcb->connector = outp->outp.info.conn;
2955		outp->dcb->i2c_index = outp->outp.info.ddc;
2956
2957		switch (outp->outp.info.type) {
2958		case NVIF_OUTP_DAC : ret = nv50_dac_create(outp); break;
2959		case NVIF_OUTP_SOR : ret = nv50_sor_create(outp); break;
2960		case NVIF_OUTP_PIOR: ret = nv50_pior_create(outp); break;
2961		default:
2962			WARN_ON(1);
2963			continue;
2964		}
2965
2966		if (ret) {
2967			NV_WARN(drm, "failed to create encoder %d/%d/%d: %d\n",
2968				i, outp->outp.info.type, outp->outp.info.proto, ret);
2969		}
2970	}
2971
2972	/* cull any connectors we created that don't have an encoder */
2973	list_for_each_entry_safe(connector, tmp, &dev->mode_config.connector_list, head) {
2974		if (connector->possible_encoders)
2975			continue;
2976
2977		NV_WARN(drm, "%s has no encoders, removing\n",
2978			connector->name);
2979		connector->funcs->destroy(connector);
2980	}
2981
2982	/* create crtc objects to represent the hw heads */
2983	for_each_set_bit(i, &disp->disp->head_mask, sizeof(disp->disp->head_mask) * 8) {
2984		struct nv50_head *head;
2985
2986		head = nv50_head_create(dev, i);
2987		if (IS_ERR(head)) {
2988			ret = PTR_ERR(head);
2989			goto out;
2990		}
2991
2992		if (has_mst) {
2993			head->msto = nv50_msto_new(dev, head, i);
2994			if (IS_ERR(head->msto)) {
2995				ret = PTR_ERR(head->msto);
2996				head->msto = NULL;
2997				goto out;
2998			}
2999
3000			/*
3001			 * FIXME: This is a hack to workaround the following
3002			 * issues:
3003			 *
3004			 * https://gitlab.gnome.org/GNOME/mutter/issues/759
3005			 * https://gitlab.freedesktop.org/xorg/xserver/merge_requests/277
3006			 *
3007			 * Once these issues are closed, this should be
3008			 * removed
3009			 */
3010			head->msto->encoder.possible_crtcs = disp->disp->head_mask;
3011		}
3012	}
3013
3014	/* Disable vblank irqs aggressively for power-saving, safe on nv50+ */
3015	dev->vblank_disable_immediate = true;
3016
3017	nv50_audio_component_init(drm);
3018
3019out:
3020	if (ret)
3021		nv50_display_destroy(dev);
3022	return ret;
3023}
3024
3025/******************************************************************************
3026 * Format modifiers
3027 *****************************************************************************/
3028
3029/****************************************************************
3030 *            Log2(block height) ----------------------------+  *
3031 *            Page Kind ----------------------------------+  |  *
3032 *            Gob Height/Page Kind Generation ------+     |  |  *
3033 *                          Sector layout -------+  |     |  |  *
3034 *                          Compression ------+  |  |     |  |  */
3035const u64 disp50xx_modifiers[] = { /*         |  |  |     |  |  */
3036	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x7a, 0),
3037	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x7a, 1),
3038	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x7a, 2),
3039	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x7a, 3),
3040	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x7a, 4),
3041	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x7a, 5),
3042	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x78, 0),
3043	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x78, 1),
3044	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x78, 2),
3045	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x78, 3),
3046	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x78, 4),
3047	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x78, 5),
3048	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x70, 0),
3049	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x70, 1),
3050	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x70, 2),
3051	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x70, 3),
3052	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x70, 4),
3053	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x70, 5),
3054	DRM_FORMAT_MOD_LINEAR,
3055	DRM_FORMAT_MOD_INVALID
3056};
3057
3058/****************************************************************
3059 *            Log2(block height) ----------------------------+  *
3060 *            Page Kind ----------------------------------+  |  *
3061 *            Gob Height/Page Kind Generation ------+     |  |  *
3062 *                          Sector layout -------+  |     |  |  *
3063 *                          Compression ------+  |  |     |  |  */
3064const u64 disp90xx_modifiers[] = { /*         |  |  |     |  |  */
3065	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 0, 0xfe, 0),
3066	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 0, 0xfe, 1),
3067	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 0, 0xfe, 2),
3068	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 0, 0xfe, 3),
3069	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 0, 0xfe, 4),
3070	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 0, 0xfe, 5),
3071	DRM_FORMAT_MOD_LINEAR,
3072	DRM_FORMAT_MOD_INVALID
3073};