FILE
drm_vblank.c
File preview with syntax highlighting.
linux-6.19.14/drivers/gpu/drm/drm_vblank.c
1/* 2 * drm_irq.c IRQ and vblank support 3 * 4 * \author Rickard E. (Rik) Faith <faith@valinux.com> 5 * \author Gareth Hughes <gareth@valinux.com> 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the "Software"), 9 * to deal in the Software without restriction, including without limitation 10 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 11 * and/or sell copies of the Software, and to permit persons to whom the 12 * Software is furnished to do so, subject to the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the next 15 * paragraph) shall be included in all copies or substantial portions of the 16 * Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 21 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 * OTHER DEALINGS IN THE SOFTWARE. 25 */ 26 27#include <linux/export.h> 28#include <linux/kthread.h> 29#include <linux/moduleparam.h> 30 31#include <drm/drm_crtc.h> 32#include <drm/drm_drv.h> 33#include <drm/drm_framebuffer.h> 34#include <drm/drm_managed.h> 35#include <drm/drm_modeset_helper_vtables.h> 36#include <drm/drm_print.h> 37#include <drm/drm_vblank.h> 38 39#include "drm_internal.h" 40#include "drm_trace.h" 41 42/** 43 * DOC: vblank handling 44 * 45 * From the computer's perspective, every time the monitor displays 46 * a new frame the scanout engine has "scanned out" the display image 47 * from top to bottom, one row of pixels at a time. The current row 48 * of pixels is referred to as the current scanline. 49 * 50 * In addition to the display's visible area, there's usually a couple of 51 * extra scanlines which aren't actually displayed on the screen. 52 * These extra scanlines don't contain image data and are occasionally used 53 * for features like audio and infoframes. The region made up of these 54 * scanlines is referred to as the vertical blanking region, or vblank for 55 * short. 56 * 57 * For historical reference, the vertical blanking period was designed to 58 * give the electron gun (on CRTs) enough time to move back to the top of 59 * the screen to start scanning out the next frame. Similar for horizontal 60 * blanking periods. They were designed to give the electron gun enough 61 * time to move back to the other side of the screen to start scanning the 62 * next scanline. 63 * 64 * :: 65 * 66 * 67 * physical → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽ 68 * top of | | 69 * display | | 70 * | New frame | 71 * | | 72 * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓| 73 * |~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| ← Scanline, 74 * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓| updates the 75 * | | frame as it 76 * | | travels down 77 * | | ("scan out") 78 * | Old frame | 79 * | | 80 * | | 81 * | | 82 * | | physical 83 * | | bottom of 84 * vertical |⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽| ← display 85 * blanking ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆ 86 * region → ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆ 87 * ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆ 88 * start of → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽ 89 * new frame 90 * 91 * "Physical top of display" is the reference point for the high-precision/ 92 * corrected timestamp. 93 * 94 * On a lot of display hardware, programming needs to take effect during the 95 * vertical blanking period so that settings like gamma, the image buffer 96 * buffer to be scanned out, etc. can safely be changed without showing 97 * any visual artifacts on the screen. In some unforgiving hardware, some of 98 * this programming has to both start and end in the same vblank. To help 99 * with the timing of the hardware programming, an interrupt is usually 100 * available to notify the driver when it can start the updating of registers. 101 * The interrupt is in this context named the vblank interrupt. 102 * 103 * The vblank interrupt may be fired at different points depending on the 104 * hardware. Some hardware implementations will fire the interrupt when the 105 * new frame start, other implementations will fire the interrupt at different 106 * points in time. 107 * 108 * Vertical blanking plays a major role in graphics rendering. To achieve 109 * tear-free display, users must synchronize page flips and/or rendering to 110 * vertical blanking. The DRM API offers ioctls to perform page flips 111 * synchronized to vertical blanking and wait for vertical blanking. 112 * 113 * The DRM core handles most of the vertical blanking management logic, which 114 * involves filtering out spurious interrupts, keeping race-free blanking 115 * counters, coping with counter wrap-around and resets and keeping use counts. 116 * It relies on the driver to generate vertical blanking interrupts and 117 * optionally provide a hardware vertical blanking counter. 118 * 119 * Drivers must initialize the vertical blanking handling core with a call to 120 * drm_vblank_init(). Minimally, a driver needs to implement 121 * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call 122 * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank 123 * support. 124 * 125 * Vertical blanking interrupts can be enabled by the DRM core or by drivers 126 * themselves (for instance to handle page flipping operations). The DRM core 127 * maintains a vertical blanking use count to ensure that the interrupts are not 128 * disabled while a user still needs them. To increment the use count, drivers 129 * call drm_crtc_vblank_get() and release the vblank reference again with 130 * drm_crtc_vblank_put(). In between these two calls vblank interrupts are 131 * guaranteed to be enabled. 132 * 133 * On many hardware disabling the vblank interrupt cannot be done in a race-free 134 * manner, see &drm_vblank_crtc_config.disable_immediate and 135 * &drm_driver.max_vblank_count. In that case the vblank core only disables the 136 * vblanks after a timer has expired, which can be configured through the 137 * ``vblankoffdelay`` module parameter. 138 * 139 * Drivers for hardware without support for vertical-blanking interrupts can 140 * use DRM vblank timers to send vblank events at the rate of the current 141 * display mode's refresh. While not synchronized to the hardware's 142 * vertical-blanking regions, the timer helps DRM clients and compositors to 143 * adapt their update cycle to the display output. Drivers should set up 144 * vblanking as usual, but call drm_crtc_vblank_start_timer() and 145 * drm_crtc_vblank_cancel_timer() as part of their atomic mode setting. 146 * See also DRM vblank helpers for more information. 147 * 148 * Drivers without support for vertical-blanking interrupts nor timers must 149 * not call drm_vblank_init(). For these drivers, atomic helpers will 150 * automatically generate fake vblank events as part of the display update. 151 * This functionality also can be controlled by the driver by enabling and 152 * disabling struct drm_crtc_state.no_vblank. 153 */ 154 155/* Retry timestamp calculation up to 3 times to satisfy 156 * drm_timestamp_precision before giving up. 157 */ 158#define DRM_TIMESTAMP_MAXRETRIES 3 159 160/* Threshold in nanoseconds for detection of redundant 161 * vblank irq in drm_handle_vblank(). 1 msec should be ok. 162 */ 163#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000 164 165static bool 166drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe, 167 ktime_t *tvblank, bool in_vblank_irq); 168 169static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */ 170 171static int drm_vblank_offdelay = 5000; /* Default to 5000 msecs. */ 172 173module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600); 174module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600); 175MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)"); 176MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]"); 177 178static struct drm_vblank_crtc * 179drm_vblank_crtc(struct drm_device *dev, unsigned int pipe) 180{ 181 return &dev->vblank[pipe]; 182} 183 184struct drm_vblank_crtc * 185drm_crtc_vblank_crtc(struct drm_crtc *crtc) 186{ 187 return drm_vblank_crtc(crtc->dev, drm_crtc_index(crtc)); 188} 189EXPORT_SYMBOL(drm_crtc_vblank_crtc); 190 191static void store_vblank(struct drm_device *dev, unsigned int pipe, 192 u32 vblank_count_inc, 193 ktime_t t_vblank, u32 last) 194{ 195 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe); 196 197 assert_spin_locked(&dev->vblank_time_lock); 198 199 vblank->last = last; 200 201 write_seqlock(&vblank->seqlock); 202 vblank->time = t_vblank; 203 atomic64_add(vblank_count_inc, &vblank->count); 204 write_sequnlock(&vblank->seqlock); 205} 206 207static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe) 208{ 209 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe); 210 211 return vblank->max_vblank_count ?: dev->max_vblank_count; 212} 213 214/* 215 * "No hw counter" fallback implementation of .get_vblank_counter() hook, 216 * if there is no usable hardware frame counter available. 217 */ 218static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe) 219{ 220 drm_WARN_ON_ONCE(dev, drm_max_vblank_count(dev, pipe) != 0); 221 return 0; 222} 223 224static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe) 225{ 226 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 227 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 228 229 if (drm_WARN_ON(dev, !crtc)) 230 return 0; 231 232 if (crtc->funcs->get_vblank_counter) 233 return crtc->funcs->get_vblank_counter(crtc); 234 } 235 236 return drm_vblank_no_hw_counter(dev, pipe); 237} 238 239/* 240 * Reset the stored timestamp for the current vblank count to correspond 241 * to the last vblank occurred. 242 * 243 * Only to be called from drm_crtc_vblank_on(). 244 * 245 * Note: caller must hold &drm_device.vbl_lock since this reads & writes 246 * device vblank fields. 247 */ 248static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe) 249{ 250 u32 cur_vblank; 251 bool rc; 252 ktime_t t_vblank; 253 int count = DRM_TIMESTAMP_MAXRETRIES; 254 255 spin_lock(&dev->vblank_time_lock); 256 257 /* 258 * sample the current counter to avoid random jumps 259 * when drm_vblank_enable() applies the diff 260 */ 261 do { 262 cur_vblank = __get_vblank_counter(dev, pipe); 263 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false); 264 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0); 265 266 /* 267 * Only reinitialize corresponding vblank timestamp if high-precision query 268 * available and didn't fail. Otherwise reinitialize delayed at next vblank 269 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid. 270 */ 271 if (!rc) 272 t_vblank = 0; 273 274 /* 275 * +1 to make sure user will never see the same 276 * vblank counter value before and after a modeset 277 */ 278 store_vblank(dev, pipe, 1, t_vblank, cur_vblank); 279 280 spin_unlock(&dev->vblank_time_lock); 281} 282 283/* 284 * Call back into the driver to update the appropriate vblank counter 285 * (specified by @pipe). Deal with wraparound, if it occurred, and 286 * update the last read value so we can deal with wraparound on the next 287 * call if necessary. 288 * 289 * Only necessary when going from off->on, to account for frames we 290 * didn't get an interrupt for. 291 * 292 * Note: caller must hold &drm_device.vbl_lock since this reads & writes 293 * device vblank fields. 294 */ 295static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe, 296 bool in_vblank_irq) 297{ 298 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe); 299 u32 cur_vblank, diff; 300 bool rc; 301 ktime_t t_vblank; 302 int count = DRM_TIMESTAMP_MAXRETRIES; 303 int framedur_ns = vblank->framedur_ns; 304 u32 max_vblank_count = drm_max_vblank_count(dev, pipe); 305 306 /* 307 * Interrupts were disabled prior to this call, so deal with counter 308 * wrap if needed. 309 * NOTE! It's possible we lost a full dev->max_vblank_count + 1 events 310 * here if the register is small or we had vblank interrupts off for 311 * a long time. 312 * 313 * We repeat the hardware vblank counter & timestamp query until 314 * we get consistent results. This to prevent races between gpu 315 * updating its hardware counter while we are retrieving the 316 * corresponding vblank timestamp. 317 */ 318 do { 319 cur_vblank = __get_vblank_counter(dev, pipe); 320 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq); 321 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0); 322 323 if (max_vblank_count) { 324 /* trust the hw counter when it's around */ 325 diff = (cur_vblank - vblank->last) & max_vblank_count; 326 } else if (rc && framedur_ns) { 327 u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time)); 328 329 /* 330 * Figure out how many vblanks we've missed based 331 * on the difference in the timestamps and the 332 * frame/field duration. 333 */ 334 335 drm_dbg_vbl(dev, "crtc %u: Calculating number of vblanks." 336 " diff_ns = %lld, framedur_ns = %d)\n", 337 pipe, (long long)diff_ns, framedur_ns); 338 339 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns); 340 341 if (diff == 0 && in_vblank_irq) 342 drm_dbg_vbl(dev, "crtc %u: Redundant vblirq ignored\n", 343 pipe); 344 } else { 345 /* some kind of default for drivers w/o accurate vbl timestamping */ 346 diff = in_vblank_irq ? 1 : 0; 347 } 348 349 /* 350 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset 351 * interval? If so then vblank irqs keep running and it will likely 352 * happen that the hardware vblank counter is not trustworthy as it 353 * might reset at some point in that interval and vblank timestamps 354 * are not trustworthy either in that interval. Iow. this can result 355 * in a bogus diff >> 1 which must be avoided as it would cause 356 * random large forward jumps of the software vblank counter. 357 */ 358 if (diff > 1 && (vblank->inmodeset & 0x2)) { 359 drm_dbg_vbl(dev, 360 "clamping vblank bump to 1 on crtc %u: diffr=%u" 361 " due to pre-modeset.\n", pipe, diff); 362 diff = 1; 363 } 364 365 drm_dbg_vbl(dev, "updating vblank count on crtc %u:" 366 " current=%llu, diff=%u, hw=%u hw_last=%u\n", 367 pipe, (unsigned long long)atomic64_read(&vblank->count), 368 diff, cur_vblank, vblank->last); 369 370 if (diff == 0) { 371 drm_WARN_ON_ONCE(dev, cur_vblank != vblank->last); 372 return; 373 } 374 375 /* 376 * Only reinitialize corresponding vblank timestamp if high-precision query 377 * available and didn't fail, or we were called from the vblank interrupt. 378 * Otherwise reinitialize delayed at next vblank interrupt and assign 0 379 * for now, to mark the vblanktimestamp as invalid. 380 */ 381 if (!rc && !in_vblank_irq) 382 t_vblank = 0; 383 384 store_vblank(dev, pipe, diff, t_vblank, cur_vblank); 385} 386 387u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe) 388{ 389 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe); 390 u64 count; 391 392 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 393 return 0; 394 395 count = atomic64_read(&vblank->count); 396 397 /* 398 * This read barrier corresponds to the implicit write barrier of the 399 * write seqlock in store_vblank(). Note that this is the only place 400 * where we need an explicit barrier, since all other access goes 401 * through drm_vblank_count_and_time(), which already has the required 402 * read barrier curtesy of the read seqlock. 403 */ 404 smp_rmb(); 405 406 return count; 407} 408 409/** 410 * drm_crtc_accurate_vblank_count - retrieve the master vblank counter 411 * @crtc: which counter to retrieve 412 * 413 * This function is similar to drm_crtc_vblank_count() but this function 414 * interpolates to handle a race with vblank interrupts using the high precision 415 * timestamping support. 416 * 417 * This is mostly useful for hardware that can obtain the scanout position, but 418 * doesn't have a hardware frame counter. 419 */ 420u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc) 421{ 422 struct drm_device *dev = crtc->dev; 423 unsigned int pipe = drm_crtc_index(crtc); 424 u64 vblank; 425 unsigned long flags; 426 427 drm_WARN_ONCE(dev, drm_debug_enabled(DRM_UT_VBL) && 428 !crtc->funcs->get_vblank_timestamp, 429 "This function requires support for accurate vblank timestamps."); 430 431 spin_lock_irqsave(&dev->vblank_time_lock, flags); 432 433 drm_update_vblank_count(dev, pipe, false); 434 vblank = drm_vblank_count(dev, pipe); 435 436 spin_unlock_irqrestore(&dev->vblank_time_lock, flags); 437 438 return vblank; 439} 440EXPORT_SYMBOL(drm_crtc_accurate_vblank_count); 441 442static void __disable_vblank(struct drm_device *dev, unsigned int pipe) 443{ 444 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 445 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 446 447 if (drm_WARN_ON(dev, !crtc)) 448 return; 449 450 if (crtc->funcs->disable_vblank) 451 crtc->funcs->disable_vblank(crtc); 452 } 453} 454 455/* 456 * Disable vblank irq's on crtc, make sure that last vblank count 457 * of hardware and corresponding consistent software vblank counter 458 * are preserved, even if there are any spurious vblank irq's after 459 * disable. 460 */ 461void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe) 462{ 463 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe); 464 unsigned long irqflags; 465 466 assert_spin_locked(&dev->vbl_lock); 467 468 /* Prevent vblank irq processing while disabling vblank irqs, 469 * so no updates of timestamps or count can happen after we've 470 * disabled. Needed to prevent races in case of delayed irq's. 471 */ 472 spin_lock_irqsave(&dev->vblank_time_lock, irqflags); 473 474 /* 475 * Update vblank count and disable vblank interrupts only if the 476 * interrupts were enabled. This avoids calling the ->disable_vblank() 477 * operation in atomic context with the hardware potentially runtime 478 * suspended. 479 */ 480 if (!vblank->enabled) 481 goto out; 482 483 /* 484 * Update the count and timestamp to maintain the 485 * appearance that the counter has been ticking all along until 486 * this time. This makes the count account for the entire time 487 * between drm_crtc_vblank_on() and drm_crtc_vblank_off(). 488 */ 489 drm_update_vblank_count(dev, pipe, false); 490 __disable_vblank(dev, pipe); 491 vblank->enabled = false; 492 493out: 494 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags); 495} 496 497static void vblank_disable_fn(struct timer_list *t) 498{ 499 struct drm_vblank_crtc *vblank = timer_container_of(vblank, t, 500 disable_timer); 501 struct drm_device *dev = vblank->dev; 502 unsigned int pipe = vblank->pipe; 503 unsigned long irqflags; 504 505 spin_lock_irqsave(&dev->vbl_lock, irqflags); 506 if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) { 507 drm_dbg_core(dev, "disabling vblank on crtc %u\n", pipe); 508 drm_vblank_disable_and_save(dev, pipe); 509 } 510 spin_unlock_irqrestore(&dev->vbl_lock, irqflags); 511} 512 513static void drm_vblank_init_release(struct drm_device *dev, void *ptr) 514{ 515 struct drm_vblank_crtc *vblank = ptr; 516 517 drm_WARN_ON(dev, READ_ONCE(vblank->enabled) && 518 drm_core_check_feature(dev, DRIVER_MODESET)); 519 520 if (vblank->vblank_timer.crtc) 521 hrtimer_cancel(&vblank->vblank_timer.timer); 522 523 drm_vblank_destroy_worker(vblank); 524 timer_delete_sync(&vblank->disable_timer); 525} 526 527/** 528 * drm_vblank_init - initialize vblank support 529 * @dev: DRM device 530 * @num_crtcs: number of CRTCs supported by @dev 531 * 532 * This function initializes vblank support for @num_crtcs display pipelines. 533 * Cleanup is handled automatically through a cleanup function added with 534 * drmm_add_action_or_reset(). 535 * 536 * Returns: 537 * Zero on success or a negative error code on failure. 538 */ 539int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs) 540{ 541 int ret; 542 unsigned int i; 543 544 spin_lock_init(&dev->vbl_lock); 545 spin_lock_init(&dev->vblank_time_lock); 546 547 dev->vblank = drmm_kcalloc(dev, num_crtcs, sizeof(*dev->vblank), GFP_KERNEL); 548 if (!dev->vblank) 549 return -ENOMEM; 550 551 dev->num_crtcs = num_crtcs; 552 553 for (i = 0; i < num_crtcs; i++) { 554 struct drm_vblank_crtc *vblank = &dev->vblank[i]; 555 556 vblank->dev = dev; 557 vblank->pipe = i; 558 init_waitqueue_head(&vblank->queue); 559 timer_setup(&vblank->disable_timer, vblank_disable_fn, 0); 560 seqlock_init(&vblank->seqlock); 561 562 ret = drmm_add_action_or_reset(dev, drm_vblank_init_release, 563 vblank); 564 if (ret) 565 return ret; 566 567 ret = drm_vblank_worker_init(vblank); 568 if (ret) 569 return ret; 570 } 571 572 return 0; 573} 574EXPORT_SYMBOL(drm_vblank_init); 575 576/** 577 * drm_dev_has_vblank - test if vblanking has been initialized for 578 * a device 579 * @dev: the device 580 * 581 * Drivers may call this function to test if vblank support is 582 * initialized for a device. For most hardware this means that vblanking 583 * can also be enabled. 584 * 585 * Atomic helpers use this function to initialize 586 * &drm_crtc_state.no_vblank. See also drm_atomic_helper_check_modeset(). 587 * 588 * Returns: 589 * True if vblanking has been initialized for the given device, false 590 * otherwise. 591 */ 592bool drm_dev_has_vblank(const struct drm_device *dev) 593{ 594 return dev->num_crtcs != 0; 595} 596EXPORT_SYMBOL(drm_dev_has_vblank); 597 598/** 599 * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC 600 * @crtc: which CRTC's vblank waitqueue to retrieve 601 * 602 * This function returns a pointer to the vblank waitqueue for the CRTC. 603 * Drivers can use this to implement vblank waits using wait_event() and related 604 * functions. 605 */ 606wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc) 607{ 608 return &crtc->dev->vblank[drm_crtc_index(crtc)].queue; 609} 610EXPORT_SYMBOL(drm_crtc_vblank_waitqueue); 611 612 613/** 614 * drm_calc_timestamping_constants - calculate vblank timestamp constants 615 * @crtc: drm_crtc whose timestamp constants should be updated. 616 * @mode: display mode containing the scanout timings 617 * 618 * Calculate and store various constants which are later needed by vblank and 619 * swap-completion timestamping, e.g, by 620 * drm_crtc_vblank_helper_get_vblank_timestamp(). They are derived from 621 * CRTC's true scanout timing, so they take things like panel scaling or 622 * other adjustments into account. 623 */ 624void drm_calc_timestamping_constants(struct drm_crtc *crtc, 625 const struct drm_display_mode *mode) 626{ 627 struct drm_device *dev = crtc->dev; 628 unsigned int pipe = drm_crtc_index(crtc); 629 struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc); 630 int linedur_ns = 0, framedur_ns = 0; 631 int dotclock = mode->crtc_clock; 632 633 if (!drm_dev_has_vblank(dev)) 634 return; 635 636 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 637 return; 638 639 /* Valid dotclock? */ 640 if (dotclock > 0) { 641 int frame_size = mode->crtc_htotal * mode->crtc_vtotal; 642 643 /* 644 * Convert scanline length in pixels and video 645 * dot clock to line duration and frame duration 646 * in nanoseconds: 647 */ 648 linedur_ns = div_u64((u64) mode->crtc_htotal * 1000000, dotclock); 649 framedur_ns = div_u64((u64) frame_size * 1000000, dotclock); 650 651 /* 652 * Fields of interlaced scanout modes are only half a frame duration. 653 */ 654 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 655 framedur_ns /= 2; 656 } else { 657 drm_err(dev, "crtc %u: Can't calculate constants, dotclock = 0!\n", 658 crtc->base.id); 659 } 660 661 vblank->linedur_ns = linedur_ns; 662 vblank->framedur_ns = framedur_ns; 663 drm_mode_copy(&vblank->hwmode, mode); 664 665 drm_dbg_core(dev, 666 "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n", 667 crtc->base.id, mode->crtc_htotal, 668 mode->crtc_vtotal, mode->crtc_vdisplay); 669 drm_dbg_core(dev, "crtc %u: clock %d kHz framedur %d linedur %d\n", 670 crtc->base.id, dotclock, framedur_ns, linedur_ns); 671} 672EXPORT_SYMBOL(drm_calc_timestamping_constants); 673 674/** 675 * drm_crtc_vblank_helper_get_vblank_timestamp_internal - precise vblank 676 * timestamp helper 677 * @crtc: CRTC whose vblank timestamp to retrieve 678 * @max_error: Desired maximum allowable error in timestamps (nanosecs) 679 * On return contains true maximum error of timestamp 680 * @vblank_time: Pointer to time which should receive the timestamp 681 * @in_vblank_irq: 682 * True when called from drm_crtc_handle_vblank(). Some drivers 683 * need to apply some workarounds for gpu-specific vblank irq quirks 684 * if flag is set. 685 * @get_scanout_position: 686 * Callback function to retrieve the scanout position. See 687 * @struct drm_crtc_helper_funcs.get_scanout_position. 688 * 689 * Implements calculation of exact vblank timestamps from given drm_display_mode 690 * timings and current video scanout position of a CRTC. 691 * 692 * The current implementation only handles standard video modes. For double scan 693 * and interlaced modes the driver is supposed to adjust the hardware mode 694 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to 695 * match the scanout position reported. 696 * 697 * Note that atomic drivers must call drm_calc_timestamping_constants() before 698 * enabling a CRTC. The atomic helpers already take care of that in 699 * drm_atomic_helper_calc_timestamping_constants(). 700 * 701 * Returns: 702 * Returns true on success, and false on failure, i.e. when no accurate 703 * timestamp could be acquired. 704 */ 705bool 706drm_crtc_vblank_helper_get_vblank_timestamp_internal( 707 struct drm_crtc *crtc, int *max_error, ktime_t *vblank_time, 708 bool in_vblank_irq, 709 drm_vblank_get_scanout_position_func get_scanout_position) 710{ 711 struct drm_device *dev = crtc->dev; 712 unsigned int pipe = crtc->index; 713 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 714 struct timespec64 ts_etime, ts_vblank_time; 715 ktime_t stime, etime; 716 bool vbl_status; 717 const struct drm_display_mode *mode; 718 int vpos, hpos, i; 719 int delta_ns, duration_ns; 720 721 if (pipe >= dev->num_crtcs) { 722 drm_err(dev, "Invalid crtc %u\n", pipe); 723 return false; 724 } 725 726 /* Scanout position query not supported? Should not happen. */ 727 if (!get_scanout_position) { 728 drm_err(dev, "Called from CRTC w/o get_scanout_position()!?\n"); 729 return false; 730 } 731 732 if (drm_drv_uses_atomic_modeset(dev)) 733 mode = &vblank->hwmode; 734 else 735 mode = &crtc->hwmode; 736 737 /* If mode timing undefined, just return as no-op: 738 * Happens during initial modesetting of a crtc. 739 */ 740 if (mode->crtc_clock == 0) { 741 drm_dbg_core(dev, "crtc %u: Noop due to uninitialized mode.\n", 742 pipe); 743 drm_WARN_ON_ONCE(dev, drm_drv_uses_atomic_modeset(dev)); 744 return false; 745 } 746 747 /* Get current scanout position with system timestamp. 748 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times 749 * if single query takes longer than max_error nanoseconds. 750 * 751 * This guarantees a tight bound on maximum error if 752 * code gets preempted or delayed for some reason. 753 */ 754 for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) { 755 /* 756 * Get vertical and horizontal scanout position vpos, hpos, 757 * and bounding timestamps stime, etime, pre/post query. 758 */ 759 vbl_status = get_scanout_position(crtc, in_vblank_irq, 760 &vpos, &hpos, 761 &stime, &etime, 762 mode); 763 764 /* Return as no-op if scanout query unsupported or failed. */ 765 if (!vbl_status) { 766 drm_dbg_core(dev, 767 "crtc %u : scanoutpos query failed.\n", 768 pipe); 769 return false; 770 } 771 772 /* Compute uncertainty in timestamp of scanout position query. */ 773 duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime); 774 775 /* Accept result with < max_error nsecs timing uncertainty. */ 776 if (duration_ns <= *max_error) 777 break; 778 } 779 780 /* Noisy system timing? */ 781 if (i == DRM_TIMESTAMP_MAXRETRIES) { 782 drm_dbg_core(dev, 783 "crtc %u: Noisy timestamp %d us > %d us [%d reps].\n", 784 pipe, duration_ns / 1000, *max_error / 1000, i); 785 } 786 787 /* Return upper bound of timestamp precision error. */ 788 *max_error = duration_ns; 789 790 /* Convert scanout position into elapsed time at raw_time query 791 * since start of scanout at first display scanline. delta_ns 792 * can be negative if start of scanout hasn't happened yet. 793 */ 794 delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos), 795 mode->crtc_clock); 796 797 /* Subtract time delta from raw timestamp to get final 798 * vblank_time timestamp for end of vblank. 799 */ 800 *vblank_time = ktime_sub_ns(etime, delta_ns); 801 802 if (!drm_debug_enabled(DRM_UT_VBL)) 803 return true; 804 805 ts_etime = ktime_to_timespec64(etime); 806 ts_vblank_time = ktime_to_timespec64(*vblank_time); 807 808 drm_dbg_vbl(dev, 809 "crtc %u : v p(%d,%d)@ %ptSp -> %ptSp [e %d us, %d rep]\n", 810 pipe, hpos, vpos, &ts_etime, &ts_vblank_time, 811 duration_ns / 1000, i); 812 813 return true; 814} 815EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp_internal); 816 817/** 818 * drm_crtc_vblank_helper_get_vblank_timestamp - precise vblank timestamp 819 * helper 820 * @crtc: CRTC whose vblank timestamp to retrieve 821 * @max_error: Desired maximum allowable error in timestamps (nanosecs) 822 * On return contains true maximum error of timestamp 823 * @vblank_time: Pointer to time which should receive the timestamp 824 * @in_vblank_irq: 825 * True when called from drm_crtc_handle_vblank(). Some drivers 826 * need to apply some workarounds for gpu-specific vblank irq quirks 827 * if flag is set. 828 * 829 * Implements calculation of exact vblank timestamps from given drm_display_mode 830 * timings and current video scanout position of a CRTC. This can be directly 831 * used as the &drm_crtc_funcs.get_vblank_timestamp implementation of a kms 832 * driver if &drm_crtc_helper_funcs.get_scanout_position is implemented. 833 * 834 * The current implementation only handles standard video modes. For double scan 835 * and interlaced modes the driver is supposed to adjust the hardware mode 836 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to 837 * match the scanout position reported. 838 * 839 * Note that atomic drivers must call drm_calc_timestamping_constants() before 840 * enabling a CRTC. The atomic helpers already take care of that in 841 * drm_atomic_helper_calc_timestamping_constants(). 842 * 843 * Returns: 844 * Returns true on success, and false on failure, i.e. when no accurate 845 * timestamp could be acquired. 846 */ 847bool drm_crtc_vblank_helper_get_vblank_timestamp(struct drm_crtc *crtc, 848 int *max_error, 849 ktime_t *vblank_time, 850 bool in_vblank_irq) 851{ 852 return drm_crtc_vblank_helper_get_vblank_timestamp_internal( 853 crtc, max_error, vblank_time, in_vblank_irq, 854 crtc->helper_private->get_scanout_position); 855} 856EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp); 857 858/** 859 * drm_crtc_get_last_vbltimestamp - retrieve raw timestamp for the most 860 * recent vblank interval 861 * @crtc: CRTC whose vblank timestamp to retrieve 862 * @tvblank: Pointer to target time which should receive the timestamp 863 * @in_vblank_irq: 864 * True when called from drm_crtc_handle_vblank(). Some drivers 865 * need to apply some workarounds for gpu-specific vblank irq quirks 866 * if flag is set. 867 * 868 * Fetches the system timestamp corresponding to the time of the most recent 869 * vblank interval on specified CRTC. May call into kms-driver to 870 * compute the timestamp with a high-precision GPU specific method. 871 * 872 * Returns zero if timestamp originates from uncorrected do_gettimeofday() 873 * call, i.e., it isn't very precisely locked to the true vblank. 874 * 875 * Returns: 876 * True if timestamp is considered to be very precise, false otherwise. 877 */ 878static bool 879drm_crtc_get_last_vbltimestamp(struct drm_crtc *crtc, ktime_t *tvblank, 880 bool in_vblank_irq) 881{ 882 bool ret = false; 883 884 /* Define requested maximum error on timestamps (nanoseconds). */ 885 int max_error = (int) drm_timestamp_precision * 1000; 886 887 /* Query driver if possible and precision timestamping enabled. */ 888 if (crtc && crtc->funcs->get_vblank_timestamp && max_error > 0) { 889 ret = crtc->funcs->get_vblank_timestamp(crtc, &max_error, 890 tvblank, in_vblank_irq); 891 } 892 893 /* GPU high precision timestamp query unsupported or failed. 894 * Return current monotonic/gettimeofday timestamp as best estimate. 895 */ 896 if (!ret) 897 *tvblank = ktime_get(); 898 899 return ret; 900} 901 902static bool 903drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe, 904 ktime_t *tvblank, bool in_vblank_irq) 905{ 906 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 907 908 return drm_crtc_get_last_vbltimestamp(crtc, tvblank, in_vblank_irq); 909} 910 911/** 912 * drm_crtc_vblank_count - retrieve "cooked" vblank counter value 913 * @crtc: which counter to retrieve 914 * 915 * Fetches the "cooked" vblank count value that represents the number of 916 * vblank events since the system was booted, including lost events due to 917 * modesetting activity. Note that this timer isn't correct against a racing 918 * vblank interrupt (since it only reports the software vblank counter), see 919 * drm_crtc_accurate_vblank_count() for such use-cases. 920 * 921 * Note that for a given vblank counter value drm_crtc_handle_vblank() 922 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time() 923 * provide a barrier: Any writes done before calling 924 * drm_crtc_handle_vblank() will be visible to callers of the later 925 * functions, if the vblank count is the same or a later one. 926 * 927 * See also &drm_vblank_crtc.count. 928 * 929 * Returns: 930 * The software vblank counter. 931 */ 932u64 drm_crtc_vblank_count(struct drm_crtc *crtc) 933{ 934 return drm_vblank_count(crtc->dev, drm_crtc_index(crtc)); 935} 936EXPORT_SYMBOL(drm_crtc_vblank_count); 937 938/** 939 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the 940 * system timestamp corresponding to that vblank counter value. 941 * @dev: DRM device 942 * @pipe: index of CRTC whose counter to retrieve 943 * @vblanktime: Pointer to ktime_t to receive the vblank timestamp. 944 * 945 * Fetches the "cooked" vblank count value that represents the number of 946 * vblank events since the system was booted, including lost events due to 947 * modesetting activity. Returns corresponding system timestamp of the time 948 * of the vblank interval that corresponds to the current vblank counter value. 949 * 950 * This is the legacy version of drm_crtc_vblank_count_and_time(). 951 */ 952static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe, 953 ktime_t *vblanktime) 954{ 955 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe); 956 u64 vblank_count; 957 unsigned int seq; 958 959 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) { 960 *vblanktime = 0; 961 return 0; 962 } 963 964 do { 965 seq = read_seqbegin(&vblank->seqlock); 966 vblank_count = atomic64_read(&vblank->count); 967 *vblanktime = vblank->time; 968 } while (read_seqretry(&vblank->seqlock, seq)); 969 970 return vblank_count; 971} 972 973/** 974 * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value 975 * and the system timestamp corresponding to that vblank counter value 976 * @crtc: which counter to retrieve 977 * @vblanktime: Pointer to time to receive the vblank timestamp. 978 * 979 * Fetches the "cooked" vblank count value that represents the number of 980 * vblank events since the system was booted, including lost events due to 981 * modesetting activity. Returns corresponding system timestamp of the time 982 * of the vblank interval that corresponds to the current vblank counter value. 983 * 984 * Note that for a given vblank counter value drm_crtc_handle_vblank() 985 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time() 986 * provide a barrier: Any writes done before calling 987 * drm_crtc_handle_vblank() will be visible to callers of the later 988 * functions, if the vblank count is the same or a later one. 989 * 990 * See also &drm_vblank_crtc.count. 991 */ 992u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc, 993 ktime_t *vblanktime) 994{ 995 return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc), 996 vblanktime); 997} 998EXPORT_SYMBOL(drm_crtc_vblank_count_and_time); 999 1000/** 1001 * drm_crtc_next_vblank_start - calculate the time of the next vblank 1002 * @crtc: the crtc for which to calculate next vblank time 1003 * @vblanktime: pointer to time to receive the next vblank timestamp. 1004 * 1005 * Calculate the expected time of the start of the next vblank period, 1006 * based on time of previous vblank and frame duration 1007 */ 1008int drm_crtc_next_vblank_start(struct drm_crtc *crtc, ktime_t *vblanktime) 1009{ 1010 struct drm_vblank_crtc *vblank; 1011 struct drm_display_mode *mode; 1012 u64 vblank_start; 1013 1014 if (!drm_dev_has_vblank(crtc->dev)) 1015 return -EINVAL; 1016 1017 vblank = drm_crtc_vblank_crtc(crtc); 1018 mode = &vblank->hwmode; 1019 1020 if (!vblank->framedur_ns || !vblank->linedur_ns) 1021 return -EINVAL; 1022 1023 if (!drm_crtc_get_last_vbltimestamp(crtc, vblanktime, false)) 1024 return -EINVAL; 1025 1026 vblank_start = DIV_ROUND_DOWN_ULL( 1027 (u64)vblank->framedur_ns * mode->crtc_vblank_start, 1028 mode->crtc_vtotal); 1029 *vblanktime = ktime_add(*vblanktime, ns_to_ktime(vblank_start)); 1030 1031 return 0; 1032} 1033EXPORT_SYMBOL(drm_crtc_next_vblank_start); 1034 1035static void send_vblank_event(struct drm_device *dev, 1036 struct drm_pending_vblank_event *e, 1037 u64 seq, ktime_t now) 1038{ 1039 struct timespec64 tv; 1040 1041 switch (e->event.base.type) { 1042 case DRM_EVENT_VBLANK: 1043 case DRM_EVENT_FLIP_COMPLETE: 1044 tv = ktime_to_timespec64(now); 1045 e->event.vbl.sequence = seq; 1046 /* 1047 * e->event is a user space structure, with hardcoded unsigned 1048 * 32-bit seconds/microseconds. This is safe as we always use 1049 * monotonic timestamps since linux-4.15 1050 */ 1051 e->event.vbl.tv_sec = tv.tv_sec; 1052 e->event.vbl.tv_usec = tv.tv_nsec / 1000; 1053 break; 1054 case DRM_EVENT_CRTC_SEQUENCE: 1055 if (seq) 1056 e->event.seq.sequence = seq; 1057 e->event.seq.time_ns = ktime_to_ns(now); 1058 break; 1059 } 1060 trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq); 1061 /* 1062 * Use the same timestamp for any associated fence signal to avoid 1063 * mismatch in timestamps for vsync & fence events triggered by the 1064 * same HW event. Frameworks like SurfaceFlinger in Android expects the 1065 * retire-fence timestamp to match exactly with HW vsync as it uses it 1066 * for its software vsync modeling. 1067 */ 1068 drm_send_event_timestamp_locked(dev, &e->base, now); 1069} 1070 1071/** 1072 * drm_crtc_arm_vblank_event - arm vblank event after pageflip 1073 * @crtc: the source CRTC of the vblank event 1074 * @e: the event to send 1075 * 1076 * A lot of drivers need to generate vblank events for the very next vblank 1077 * interrupt. For example when the page flip interrupt happens when the page 1078 * flip gets armed, but not when it actually executes within the next vblank 1079 * period. This helper function implements exactly the required vblank arming 1080 * behaviour. 1081 * 1082 * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an 1083 * atomic commit must ensure that the next vblank happens at exactly the same 1084 * time as the atomic commit is committed to the hardware. This function itself 1085 * does **not** protect against the next vblank interrupt racing with either this 1086 * function call or the atomic commit operation. A possible sequence could be: 1087 * 1088 * 1. Driver commits new hardware state into vblank-synchronized registers. 1089 * 2. A vblank happens, committing the hardware state. Also the corresponding 1090 * vblank interrupt is fired off and fully processed by the interrupt 1091 * handler. 1092 * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event(). 1093 * 4. The event is only send out for the next vblank, which is wrong. 1094 * 1095 * An equivalent race can happen when the driver calls 1096 * drm_crtc_arm_vblank_event() before writing out the new hardware state. 1097 * 1098 * The only way to make this work safely is to prevent the vblank from firing 1099 * (and the hardware from committing anything else) until the entire atomic 1100 * commit sequence has run to completion. If the hardware does not have such a 1101 * feature (e.g. using a "go" bit), then it is unsafe to use this functions. 1102 * Instead drivers need to manually send out the event from their interrupt 1103 * handler by calling drm_crtc_send_vblank_event() and make sure that there's no 1104 * possible race with the hardware committing the atomic update. 1105 * 1106 * Caller must hold a vblank reference for the event @e acquired by a 1107 * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives. 1108 */ 1109void drm_crtc_arm_vblank_event(struct drm_crtc *crtc, 1110 struct drm_pending_vblank_event *e) 1111{ 1112 struct drm_device *dev = crtc->dev; 1113 unsigned int pipe = drm_crtc_index(crtc); 1114 1115 assert_spin_locked(&dev->event_lock); 1116 1117 e->pipe = pipe; 1118 e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1; 1119 list_add_tail(&e->base.link, &dev->vblank_event_list); 1120} 1121EXPORT_SYMBOL(drm_crtc_arm_vblank_event); 1122 1123/** 1124 * drm_crtc_send_vblank_event - helper to send vblank event after pageflip 1125 * @crtc: the source CRTC of the vblank event 1126 * @e: the event to send 1127 * 1128 * Updates sequence # and timestamp on event for the most recently processed 1129 * vblank, and sends it to userspace. Caller must hold event lock. 1130 * 1131 * See drm_crtc_arm_vblank_event() for a helper which can be used in certain 1132 * situation, especially to send out events for atomic commit operations. 1133 */ 1134void drm_crtc_send_vblank_event(struct drm_crtc *crtc, 1135 struct drm_pending_vblank_event *e) 1136{ 1137 struct drm_device *dev = crtc->dev; 1138 u64 seq; 1139 unsigned int pipe = drm_crtc_index(crtc); 1140 ktime_t now; 1141 1142 if (drm_dev_has_vblank(dev)) { 1143 seq = drm_vblank_count_and_time(dev, pipe, &now); 1144 } else { 1145 seq = 0; 1146 1147 now = ktime_get(); 1148 } 1149 e->pipe = pipe; 1150 send_vblank_event(dev, e, seq, now); 1151} 1152EXPORT_SYMBOL(drm_crtc_send_vblank_event); 1153 1154static int __enable_vblank(struct drm_device *dev, unsigned int pipe) 1155{ 1156 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 1157 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 1158 1159 if (drm_WARN_ON(dev, !crtc)) 1160 return 0; 1161 1162 if (crtc->funcs->enable_vblank) 1163 return crtc->funcs->enable_vblank(crtc); 1164 } 1165 1166 return -EINVAL; 1167} 1168 1169static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe) 1170{ 1171 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe); 1172 int ret = 0; 1173 1174 assert_spin_locked(&dev->vbl_lock); 1175 1176 spin_lock(&dev->vblank_time_lock); 1177 1178 if (!vblank->enabled) { 1179 /* 1180 * Enable vblank irqs under vblank_time_lock protection. 1181 * All vblank count & timestamp updates are held off 1182 * until we are done reinitializing master counter and 1183 * timestamps. Filtercode in drm_handle_vblank() will 1184 * prevent double-accounting of same vblank interval. 1185 */ 1186 ret = __enable_vblank(dev, pipe); 1187 drm_dbg_core(dev, "enabling vblank on crtc %u, ret: %d\n", 1188 pipe, ret); 1189 if (ret) { 1190 atomic_dec(&vblank->refcount); 1191 } else { 1192 drm_update_vblank_count(dev, pipe, 0); 1193 /* drm_update_vblank_count() includes a wmb so we just 1194 * need to ensure that the compiler emits the write 1195 * to mark the vblank as enabled after the call 1196 * to drm_update_vblank_count(). 1197 */ 1198 WRITE_ONCE(vblank->enabled, true); 1199 } 1200 } 1201 1202 spin_unlock(&dev->vblank_time_lock); 1203 1204 return ret; 1205} 1206 1207int drm_vblank_get(struct drm_device *dev, unsigned int pipe) 1208{ 1209 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe); 1210 unsigned long irqflags; 1211 int ret = 0; 1212 1213 if (!drm_dev_has_vblank(dev)) 1214 return -EINVAL; 1215 1216 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1217 return -EINVAL; 1218 1219 spin_lock_irqsave(&dev->vbl_lock, irqflags); 1220 /* Going from 0->1 means we have to enable interrupts again */ 1221 if (atomic_add_return(1, &vblank->refcount) == 1) { 1222 ret = drm_vblank_enable(dev, pipe); 1223 } else { 1224 if (!vblank->enabled) { 1225 atomic_dec(&vblank->refcount); 1226 ret = -EINVAL; 1227 } 1228 } 1229 spin_unlock_irqrestore(&dev->vbl_lock, irqflags); 1230 1231 return ret; 1232} 1233 1234/** 1235 * drm_crtc_vblank_get - get a reference count on vblank events 1236 * @crtc: which CRTC to own 1237 * 1238 * Acquire a reference count on vblank events to avoid having them disabled 1239 * while in use. 1240 * 1241 * Returns: 1242 * Zero on success or a negative error code on failure. 1243 */ 1244int drm_crtc_vblank_get(struct drm_crtc *crtc) 1245{ 1246 return drm_vblank_get(crtc->dev, drm_crtc_index(crtc)); 1247} 1248EXPORT_SYMBOL(drm_crtc_vblank_get); 1249 1250void drm_vblank_put(struct drm_device *dev, unsigned int pipe) 1251{ 1252 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe); 1253 int vblank_offdelay = vblank->config.offdelay_ms; 1254 1255 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1256 return; 1257 1258 if (drm_WARN_ON(dev, atomic_read(&vblank->refcount) == 0)) 1259 return; 1260 1261 /* Last user schedules interrupt disable */ 1262 if (atomic_dec_and_test(&vblank->refcount)) { 1263 if (!vblank_offdelay) 1264 return; 1265 else if (vblank_offdelay < 0) 1266 vblank_disable_fn(&vblank->disable_timer); 1267 else if (!vblank->config.disable_immediate) 1268 mod_timer(&vblank->disable_timer, 1269 jiffies + ((vblank_offdelay * HZ) / 1000)); 1270 } 1271} 1272 1273/** 1274 * drm_crtc_vblank_put - give up ownership of vblank events 1275 * @crtc: which counter to give up 1276 * 1277 * Release ownership of a given vblank counter, turning off interrupts 1278 * if possible. Disable interrupts after &drm_vblank_crtc_config.offdelay_ms 1279 * milliseconds. 1280 */ 1281void drm_crtc_vblank_put(struct drm_crtc *crtc) 1282{ 1283 drm_vblank_put(crtc->dev, drm_crtc_index(crtc)); 1284} 1285EXPORT_SYMBOL(drm_crtc_vblank_put); 1286 1287/** 1288 * drm_wait_one_vblank - wait for one vblank 1289 * @dev: DRM device 1290 * @pipe: CRTC index 1291 * 1292 * This waits for one vblank to pass on @pipe, using the irq driver interfaces. 1293 * It is a failure to call this when the vblank irq for @pipe is disabled, e.g. 1294 * due to lack of driver support or because the crtc is off. 1295 * 1296 * This is the legacy version of drm_crtc_wait_one_vblank(). 1297 */ 1298void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe) 1299{ 1300 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe); 1301 int ret; 1302 u64 last; 1303 1304 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1305 return; 1306 1307 ret = drm_vblank_get(dev, pipe); 1308 if (drm_WARN(dev, ret, "vblank not available on crtc %i, ret=%i\n", 1309 pipe, ret)) 1310 return; 1311 1312 last = drm_vblank_count(dev, pipe); 1313 1314 ret = wait_event_timeout(vblank->queue, 1315 last != drm_vblank_count(dev, pipe), 1316 msecs_to_jiffies(1000)); 1317 1318 drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe); 1319 1320 drm_vblank_put(dev, pipe); 1321} 1322EXPORT_SYMBOL(drm_wait_one_vblank); 1323 1324/** 1325 * drm_crtc_wait_one_vblank - wait for one vblank 1326 * @crtc: DRM crtc 1327 * 1328 * This waits for one vblank to pass on @crtc, using the irq driver interfaces. 1329 * It is a failure to call this when the vblank irq for @crtc is disabled, e.g. 1330 * due to lack of driver support or because the crtc is off. 1331 */ 1332void drm_crtc_wait_one_vblank(struct drm_crtc *crtc) 1333{ 1334 drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc)); 1335} 1336EXPORT_SYMBOL(drm_crtc_wait_one_vblank); 1337 1338/** 1339 * drm_crtc_vblank_off - disable vblank events on a CRTC 1340 * @crtc: CRTC in question 1341 * 1342 * Drivers can use this function to shut down the vblank interrupt handling when 1343 * disabling a crtc. This function ensures that the latest vblank frame count is 1344 * stored so that drm_vblank_on can restore it again. 1345 * 1346 * Drivers must use this function when the hardware vblank counter can get 1347 * reset, e.g. when suspending or disabling the @crtc in general. 1348 */ 1349void drm_crtc_vblank_off(struct drm_crtc *crtc) 1350{ 1351 struct drm_device *dev = crtc->dev; 1352 unsigned int pipe = drm_crtc_index(crtc); 1353 struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc); 1354 struct drm_pending_vblank_event *e, *t; 1355 ktime_t now; 1356 u64 seq; 1357 1358 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1359 return; 1360 1361 /* 1362 * Grab event_lock early to prevent vblank work from being scheduled 1363 * while we're in the middle of shutting down vblank interrupts 1364 */ 1365 spin_lock_irq(&dev->event_lock); 1366 1367 spin_lock(&dev->vbl_lock); 1368 drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n", 1369 pipe, vblank->enabled, vblank->inmodeset); 1370 1371 /* Avoid redundant vblank disables without previous 1372 * drm_crtc_vblank_on(). */ 1373 if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset) 1374 drm_vblank_disable_and_save(dev, pipe); 1375 1376 wake_up(&vblank->queue); 1377 1378 /* 1379 * Prevent subsequent drm_vblank_get() from re-enabling 1380 * the vblank interrupt by bumping the refcount. 1381 */ 1382 if (!vblank->inmodeset) { 1383 atomic_inc(&vblank->refcount); 1384 vblank->inmodeset = 1; 1385 } 1386 spin_unlock(&dev->vbl_lock); 1387 1388 /* Send any queued vblank events, lest the natives grow disquiet */ 1389 seq = drm_vblank_count_and_time(dev, pipe, &now); 1390 1391 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) { 1392 if (e->pipe != pipe) 1393 continue; 1394 drm_dbg_core(dev, "Sending premature vblank event on disable: " 1395 "wanted %llu, current %llu\n", 1396 e->sequence, seq); 1397 list_del(&e->base.link); 1398 drm_vblank_put(dev, pipe); 1399 send_vblank_event(dev, e, seq, now); 1400 } 1401 1402 /* Cancel any leftover pending vblank work */ 1403 drm_vblank_cancel_pending_works(vblank); 1404 1405 spin_unlock_irq(&dev->event_lock); 1406 1407 /* Will be reset by the modeset helpers when re-enabling the crtc by 1408 * calling drm_calc_timestamping_constants(). */ 1409 vblank->hwmode.crtc_clock = 0; 1410 1411 /* Wait for any vblank work that's still executing to finish */ 1412 drm_vblank_flush_worker(vblank); 1413} 1414EXPORT_SYMBOL(drm_crtc_vblank_off); 1415 1416/** 1417 * drm_crtc_vblank_reset - reset vblank state to off on a CRTC 1418 * @crtc: CRTC in question 1419 * 1420 * Drivers can use this function to reset the vblank state to off at load time. 1421 * Drivers should use this together with the drm_crtc_vblank_off() and 1422 * drm_crtc_vblank_on() functions. The difference compared to 1423 * drm_crtc_vblank_off() is that this function doesn't save the vblank counter 1424 * and hence doesn't need to call any driver hooks. 1425 * 1426 * This is useful for recovering driver state e.g. on driver load, or on resume. 1427 */ 1428void drm_crtc_vblank_reset(struct drm_crtc *crtc) 1429{ 1430 struct drm_device *dev = crtc->dev; 1431 struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc); 1432 1433 spin_lock_irq(&dev->vbl_lock); 1434 /* 1435 * Prevent subsequent drm_vblank_get() from enabling the vblank 1436 * interrupt by bumping the refcount. 1437 */ 1438 if (!vblank->inmodeset) { 1439 atomic_inc(&vblank->refcount); 1440 vblank->inmodeset = 1; 1441 } 1442 spin_unlock_irq(&dev->vbl_lock); 1443 1444 drm_WARN_ON(dev, !list_empty(&dev->vblank_event_list)); 1445 drm_WARN_ON(dev, !list_empty(&vblank->pending_work)); 1446} 1447EXPORT_SYMBOL(drm_crtc_vblank_reset); 1448 1449/** 1450 * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value 1451 * @crtc: CRTC in question 1452 * @max_vblank_count: max hardware vblank counter value 1453 * 1454 * Update the maximum hardware vblank counter value for @crtc 1455 * at runtime. Useful for hardware where the operation of the 1456 * hardware vblank counter depends on the currently active 1457 * display configuration. 1458 * 1459 * For example, if the hardware vblank counter does not work 1460 * when a specific connector is active the maximum can be set 1461 * to zero. And when that specific connector isn't active the 1462 * maximum can again be set to the appropriate non-zero value. 1463 * 1464 * If used, must be called before drm_vblank_on(). 1465 */ 1466void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc, 1467 u32 max_vblank_count) 1468{ 1469 struct drm_device *dev = crtc->dev; 1470 struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc); 1471 1472 drm_WARN_ON(dev, dev->max_vblank_count); 1473 drm_WARN_ON(dev, !READ_ONCE(vblank->inmodeset)); 1474 1475 vblank->max_vblank_count = max_vblank_count; 1476} 1477EXPORT_SYMBOL(drm_crtc_set_max_vblank_count); 1478 1479/** 1480 * drm_crtc_vblank_on_config - enable vblank events on a CRTC with custom 1481 * configuration options 1482 * @crtc: CRTC in question 1483 * @config: Vblank configuration value 1484 * 1485 * See drm_crtc_vblank_on(). In addition, this function allows you to provide a 1486 * custom vblank configuration for a given CRTC. 1487 * 1488 * Note that @config is copied, the pointer does not need to stay valid beyond 1489 * this function call. For details of the parameters see 1490 * struct drm_vblank_crtc_config. 1491 */ 1492void drm_crtc_vblank_on_config(struct drm_crtc *crtc, 1493 const struct drm_vblank_crtc_config *config) 1494{ 1495 struct drm_device *dev = crtc->dev; 1496 unsigned int pipe = drm_crtc_index(crtc); 1497 struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc); 1498 1499 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1500 return; 1501 1502 spin_lock_irq(&dev->vbl_lock); 1503 drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n", 1504 pipe, vblank->enabled, vblank->inmodeset); 1505 1506 vblank->config = *config; 1507 1508 /* Drop our private "prevent drm_vblank_get" refcount */ 1509 if (vblank->inmodeset) { 1510 atomic_dec(&vblank->refcount); 1511 vblank->inmodeset = 0; 1512 } 1513 1514 drm_reset_vblank_timestamp(dev, pipe); 1515 1516 /* 1517 * re-enable interrupts if there are users left, or the 1518 * user wishes vblank interrupts to be enabled all the time. 1519 */ 1520 if (atomic_read(&vblank->refcount) != 0 || !vblank->config.offdelay_ms) 1521 drm_WARN_ON(dev, drm_vblank_enable(dev, pipe)); 1522 spin_unlock_irq(&dev->vbl_lock); 1523} 1524EXPORT_SYMBOL(drm_crtc_vblank_on_config); 1525 1526/** 1527 * drm_crtc_vblank_on - enable vblank events on a CRTC 1528 * @crtc: CRTC in question 1529 * 1530 * This functions restores the vblank interrupt state captured with 1531 * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note 1532 * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be 1533 * unbalanced and so can also be unconditionally called in driver load code to 1534 * reflect the current hardware state of the crtc. 1535 * 1536 * Note that unlike in drm_crtc_vblank_on_config(), default values are used. 1537 */ 1538void drm_crtc_vblank_on(struct drm_crtc *crtc) 1539{ 1540 const struct drm_vblank_crtc_config config = { 1541 .offdelay_ms = drm_vblank_offdelay, 1542 .disable_immediate = crtc->dev->vblank_disable_immediate 1543 }; 1544 1545 drm_crtc_vblank_on_config(crtc, &config); 1546} 1547EXPORT_SYMBOL(drm_crtc_vblank_on); 1548 1549static void drm_vblank_restore(struct drm_device *dev, unsigned int pipe) 1550{ 1551 ktime_t t_vblank; 1552 struct drm_vblank_crtc *vblank; 1553 int framedur_ns; 1554 u64 diff_ns; 1555 u32 cur_vblank, diff = 1; 1556 int count = DRM_TIMESTAMP_MAXRETRIES; 1557 u32 max_vblank_count = drm_max_vblank_count(dev, pipe); 1558 1559 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1560 return; 1561 1562 assert_spin_locked(&dev->vbl_lock); 1563 assert_spin_locked(&dev->vblank_time_lock); 1564 1565 vblank = drm_vblank_crtc(dev, pipe); 1566 drm_WARN_ONCE(dev, 1567 drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns, 1568 "Cannot compute missed vblanks without frame duration\n"); 1569 framedur_ns = vblank->framedur_ns; 1570 1571 do { 1572 cur_vblank = __get_vblank_counter(dev, pipe); 1573 drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false); 1574 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0); 1575 1576 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time)); 1577 if (framedur_ns) 1578 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns); 1579 1580 1581 drm_dbg_vbl(dev, 1582 "missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n", 1583 diff, diff_ns, framedur_ns, cur_vblank - vblank->last); 1584 vblank->last = (cur_vblank - diff) & max_vblank_count; 1585} 1586 1587/** 1588 * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count. 1589 * @crtc: CRTC in question 1590 * 1591 * Power manamement features can cause frame counter resets between vblank 1592 * disable and enable. Drivers can use this function in their 1593 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since 1594 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the 1595 * vblank counter. 1596 * 1597 * Note that drivers must have race-free high-precision timestamping support, 1598 * i.e. &drm_crtc_funcs.get_vblank_timestamp must be hooked up and 1599 * &drm_vblank_crtc_config.disable_immediate must be set to indicate the 1600 * time-stamping functions are race-free against vblank hardware counter 1601 * increments. 1602 */ 1603void drm_crtc_vblank_restore(struct drm_crtc *crtc) 1604{ 1605 struct drm_device *dev = crtc->dev; 1606 unsigned int pipe = drm_crtc_index(crtc); 1607 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe); 1608 1609 drm_WARN_ON_ONCE(dev, !crtc->funcs->get_vblank_timestamp); 1610 drm_WARN_ON_ONCE(dev, vblank->inmodeset); 1611 drm_WARN_ON_ONCE(dev, !vblank->config.disable_immediate); 1612 1613 drm_vblank_restore(dev, pipe); 1614} 1615EXPORT_SYMBOL(drm_crtc_vblank_restore); 1616 1617static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe, 1618 u64 req_seq, 1619 union drm_wait_vblank *vblwait, 1620 struct drm_file *file_priv) 1621{ 1622 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe); 1623 struct drm_pending_vblank_event *e; 1624 ktime_t now; 1625 u64 seq; 1626 int ret; 1627 1628 e = kzalloc(sizeof(*e), GFP_KERNEL); 1629 if (e == NULL) { 1630 ret = -ENOMEM; 1631 goto err_put; 1632 } 1633 1634 e->pipe = pipe; 1635 e->event.base.type = DRM_EVENT_VBLANK; 1636 e->event.base.length = sizeof(e->event.vbl); 1637 e->event.vbl.user_data = vblwait->request.signal; 1638 e->event.vbl.crtc_id = 0; 1639 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 1640 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 1641 1642 if (crtc) 1643 e->event.vbl.crtc_id = crtc->base.id; 1644 } 1645 1646 spin_lock_irq(&dev->event_lock); 1647 1648 /* 1649 * drm_crtc_vblank_off() might have been called after we called 1650 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the 1651 * vblank disable, so no need for further locking. The reference from 1652 * drm_vblank_get() protects against vblank disable from another source. 1653 */ 1654 if (!READ_ONCE(vblank->enabled)) { 1655 ret = -EINVAL; 1656 goto err_unlock; 1657 } 1658 1659 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base, 1660 &e->event.base); 1661 1662 if (ret) 1663 goto err_unlock; 1664 1665 seq = drm_vblank_count_and_time(dev, pipe, &now); 1666 1667 drm_dbg_core(dev, "event on vblank count %llu, current %llu, crtc %u\n", 1668 req_seq, seq, pipe); 1669 1670 trace_drm_vblank_event_queued(file_priv, pipe, req_seq); 1671 1672 e->sequence = req_seq; 1673 if (drm_vblank_passed(seq, req_seq)) { 1674 drm_vblank_put(dev, pipe); 1675 send_vblank_event(dev, e, seq, now); 1676 vblwait->reply.sequence = seq; 1677 } else { 1678 /* drm_handle_vblank_events will call drm_vblank_put */ 1679 list_add_tail(&e->base.link, &dev->vblank_event_list); 1680 vblwait->reply.sequence = req_seq; 1681 } 1682 1683 spin_unlock_irq(&dev->event_lock); 1684 1685 return 0; 1686 1687err_unlock: 1688 spin_unlock_irq(&dev->event_lock); 1689 kfree(e); 1690err_put: 1691 drm_vblank_put(dev, pipe); 1692 return ret; 1693} 1694 1695static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait) 1696{ 1697 if (vblwait->request.sequence) 1698 return false; 1699 1700 return _DRM_VBLANK_RELATIVE == 1701 (vblwait->request.type & (_DRM_VBLANK_TYPES_MASK | 1702 _DRM_VBLANK_EVENT | 1703 _DRM_VBLANK_NEXTONMISS)); 1704} 1705 1706/* 1707 * Widen a 32-bit param to 64-bits. 1708 * 1709 * \param narrow 32-bit value (missing upper 32 bits) 1710 * \param near 64-bit value that should be 'close' to near 1711 * 1712 * This function returns a 64-bit value using the lower 32-bits from 1713 * 'narrow' and constructing the upper 32-bits so that the result is 1714 * as close as possible to 'near'. 1715 */ 1716 1717static u64 widen_32_to_64(u32 narrow, u64 near) 1718{ 1719 return near + (s32) (narrow - near); 1720} 1721 1722static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe, 1723 struct drm_wait_vblank_reply *reply) 1724{ 1725 ktime_t now; 1726 struct timespec64 ts; 1727 1728 /* 1729 * drm_wait_vblank_reply is a UAPI structure that uses 'long' 1730 * to store the seconds. This is safe as we always use monotonic 1731 * timestamps since linux-4.15. 1732 */ 1733 reply->sequence = drm_vblank_count_and_time(dev, pipe, &now); 1734 ts = ktime_to_timespec64(now); 1735 reply->tval_sec = (u32)ts.tv_sec; 1736 reply->tval_usec = ts.tv_nsec / 1000; 1737} 1738 1739static bool drm_wait_vblank_supported(struct drm_device *dev) 1740{ 1741 return drm_dev_has_vblank(dev); 1742} 1743 1744int drm_wait_vblank_ioctl(struct drm_device *dev, void *data, 1745 struct drm_file *file_priv) 1746{ 1747 struct drm_crtc *crtc; 1748 struct drm_vblank_crtc *vblank; 1749 union drm_wait_vblank *vblwait = data; 1750 int ret; 1751 u64 req_seq, seq; 1752 unsigned int pipe_index; 1753 unsigned int flags, pipe, high_pipe; 1754 1755 if (!drm_wait_vblank_supported(dev)) 1756 return -EOPNOTSUPP; 1757 1758 if (vblwait->request.type & _DRM_VBLANK_SIGNAL) 1759 return -EINVAL; 1760 1761 if (vblwait->request.type & 1762 ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK | 1763 _DRM_VBLANK_HIGH_CRTC_MASK)) { 1764 drm_dbg_core(dev, 1765 "Unsupported type value 0x%x, supported mask 0x%x\n", 1766 vblwait->request.type, 1767 (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK | 1768 _DRM_VBLANK_HIGH_CRTC_MASK)); 1769 return -EINVAL; 1770 } 1771 1772 flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK; 1773 high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK); 1774 if (high_pipe) 1775 pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT; 1776 else 1777 pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0; 1778 1779 /* Convert lease-relative crtc index into global crtc index */ 1780 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 1781 pipe = 0; 1782 drm_for_each_crtc(crtc, dev) { 1783 if (drm_lease_held(file_priv, crtc->base.id)) { 1784 if (pipe_index == 0) 1785 break; 1786 pipe_index--; 1787 } 1788 pipe++; 1789 } 1790 } else { 1791 pipe = pipe_index; 1792 } 1793 1794 if (pipe >= dev->num_crtcs) 1795 return -EINVAL; 1796 1797 vblank = &dev->vblank[pipe]; 1798 1799 /* If the counter is currently enabled and accurate, short-circuit 1800 * queries to return the cached timestamp of the last vblank. 1801 */ 1802 if (vblank->config.disable_immediate && 1803 drm_wait_vblank_is_query(vblwait) && 1804 READ_ONCE(vblank->enabled)) { 1805 drm_wait_vblank_reply(dev, pipe, &vblwait->reply); 1806 return 0; 1807 } 1808 1809 ret = drm_vblank_get(dev, pipe); 1810 if (ret) { 1811 drm_dbg_core(dev, 1812 "crtc %d failed to acquire vblank counter, %d\n", 1813 pipe, ret); 1814 return ret; 1815 } 1816 seq = drm_vblank_count(dev, pipe); 1817 1818 switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) { 1819 case _DRM_VBLANK_RELATIVE: 1820 req_seq = seq + vblwait->request.sequence; 1821 vblwait->request.sequence = req_seq; 1822 vblwait->request.type &= ~_DRM_VBLANK_RELATIVE; 1823 break; 1824 case _DRM_VBLANK_ABSOLUTE: 1825 req_seq = widen_32_to_64(vblwait->request.sequence, seq); 1826 break; 1827 default: 1828 ret = -EINVAL; 1829 goto done; 1830 } 1831 1832 if ((flags & _DRM_VBLANK_NEXTONMISS) && 1833 drm_vblank_passed(seq, req_seq)) { 1834 req_seq = seq + 1; 1835 vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS; 1836 vblwait->request.sequence = req_seq; 1837 } 1838 1839 if (flags & _DRM_VBLANK_EVENT) { 1840 /* must hold on to the vblank ref until the event fires 1841 * drm_vblank_put will be called asynchronously 1842 */ 1843 return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv); 1844 } 1845 1846 if (req_seq != seq) { 1847 int wait; 1848 1849 drm_dbg_core(dev, "waiting on vblank count %llu, crtc %u\n", 1850 req_seq, pipe); 1851 wait = wait_event_interruptible_timeout(vblank->queue, 1852 drm_vblank_passed(drm_vblank_count(dev, pipe), req_seq) || 1853 !READ_ONCE(vblank->enabled), 1854 msecs_to_jiffies(3000)); 1855 1856 switch (wait) { 1857 case 0: 1858 /* timeout */ 1859 ret = -EBUSY; 1860 break; 1861 case -ERESTARTSYS: 1862 /* interrupted by signal */ 1863 ret = -EINTR; 1864 break; 1865 default: 1866 ret = 0; 1867 break; 1868 } 1869 } 1870 1871 if (ret != -EINTR) { 1872 drm_wait_vblank_reply(dev, pipe, &vblwait->reply); 1873 1874 drm_dbg_core(dev, "crtc %d returning %u to client\n", 1875 pipe, vblwait->reply.sequence); 1876 } else { 1877 drm_dbg_core(dev, "crtc %d vblank wait interrupted by signal\n", 1878 pipe); 1879 } 1880 1881done: 1882 drm_vblank_put(dev, pipe); 1883 return ret; 1884} 1885 1886static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe) 1887{ 1888 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 1889 bool high_prec = false; 1890 struct drm_pending_vblank_event *e, *t; 1891 ktime_t now; 1892 u64 seq; 1893 1894 assert_spin_locked(&dev->event_lock); 1895 1896 seq = drm_vblank_count_and_time(dev, pipe, &now); 1897 1898 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) { 1899 if (e->pipe != pipe) 1900 continue; 1901 if (!drm_vblank_passed(seq, e->sequence)) 1902 continue; 1903 1904 drm_dbg_core(dev, "vblank event on %llu, current %llu\n", 1905 e->sequence, seq); 1906 1907 list_del(&e->base.link); 1908 drm_vblank_put(dev, pipe); 1909 send_vblank_event(dev, e, seq, now); 1910 } 1911 1912 if (crtc && crtc->funcs->get_vblank_timestamp) 1913 high_prec = true; 1914 1915 trace_drm_vblank_event(pipe, seq, now, high_prec); 1916} 1917 1918/** 1919 * drm_handle_vblank - handle a vblank event 1920 * @dev: DRM device 1921 * @pipe: index of CRTC where this event occurred 1922 * 1923 * Drivers should call this routine in their vblank interrupt handlers to 1924 * update the vblank counter and send any signals that may be pending. 1925 * 1926 * This is the legacy version of drm_crtc_handle_vblank(). 1927 */ 1928bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe) 1929{ 1930 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe); 1931 unsigned long irqflags; 1932 bool disable_irq; 1933 1934 if (drm_WARN_ON_ONCE(dev, !drm_dev_has_vblank(dev))) 1935 return false; 1936 1937 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1938 return false; 1939 1940 spin_lock_irqsave(&dev->event_lock, irqflags); 1941 1942 /* Need timestamp lock to prevent concurrent execution with 1943 * vblank enable/disable, as this would cause inconsistent 1944 * or corrupted timestamps and vblank counts. 1945 */ 1946 spin_lock(&dev->vblank_time_lock); 1947 1948 /* Vblank irq handling disabled. Nothing to do. */ 1949 if (!vblank->enabled) { 1950 spin_unlock(&dev->vblank_time_lock); 1951 spin_unlock_irqrestore(&dev->event_lock, irqflags); 1952 return false; 1953 } 1954 1955 drm_update_vblank_count(dev, pipe, true); 1956 1957 spin_unlock(&dev->vblank_time_lock); 1958 1959 wake_up(&vblank->queue); 1960 1961 /* With instant-off, we defer disabling the interrupt until after 1962 * we finish processing the following vblank after all events have 1963 * been signaled. The disable has to be last (after 1964 * drm_handle_vblank_events) so that the timestamp is always accurate. 1965 */ 1966 disable_irq = (vblank->config.disable_immediate && 1967 vblank->config.offdelay_ms > 0 && 1968 !atomic_read(&vblank->refcount)); 1969 1970 drm_handle_vblank_events(dev, pipe); 1971 drm_handle_vblank_works(vblank); 1972 1973 spin_unlock_irqrestore(&dev->event_lock, irqflags); 1974 1975 if (disable_irq) 1976 vblank_disable_fn(&vblank->disable_timer); 1977 1978 return true; 1979} 1980EXPORT_SYMBOL(drm_handle_vblank); 1981 1982/** 1983 * drm_crtc_handle_vblank - handle a vblank event 1984 * @crtc: where this event occurred 1985 * 1986 * Drivers should call this routine in their vblank interrupt handlers to 1987 * update the vblank counter and send any signals that may be pending. 1988 * 1989 * This is the native KMS version of drm_handle_vblank(). 1990 * 1991 * Note that for a given vblank counter value drm_crtc_handle_vblank() 1992 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time() 1993 * provide a barrier: Any writes done before calling 1994 * drm_crtc_handle_vblank() will be visible to callers of the later 1995 * functions, if the vblank count is the same or a later one. 1996 * 1997 * See also &drm_vblank_crtc.count. 1998 * 1999 * Returns: 2000 * True if the event was successfully handled, false on failure. 2001 */ 2002bool drm_crtc_handle_vblank(struct drm_crtc *crtc) 2003{ 2004 return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc)); 2005} 2006EXPORT_SYMBOL(drm_crtc_handle_vblank); 2007 2008/* 2009 * Get crtc VBLANK count. 2010 * 2011 * \param dev DRM device 2012 * \param data user argument, pointing to a drm_crtc_get_sequence structure. 2013 * \param file_priv drm file private for the user's open file descriptor 2014 */ 2015 2016int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data, 2017 struct drm_file *file_priv) 2018{ 2019 struct drm_crtc *crtc; 2020 struct drm_vblank_crtc *vblank; 2021 int pipe; 2022 struct drm_crtc_get_sequence *get_seq = data; 2023 ktime_t now; 2024 bool vblank_enabled; 2025 int ret; 2026 2027 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 2028 return -EOPNOTSUPP; 2029 2030 if (!drm_dev_has_vblank(dev)) 2031 return -EOPNOTSUPP; 2032 2033 crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id); 2034 if (!crtc) 2035 return -ENOENT; 2036 2037 pipe = drm_crtc_index(crtc); 2038 2039 vblank = drm_crtc_vblank_crtc(crtc); 2040 vblank_enabled = READ_ONCE(vblank->config.disable_immediate) && 2041 READ_ONCE(vblank->enabled); 2042 2043 if (!vblank_enabled) { 2044 ret = drm_crtc_vblank_get(crtc); 2045 if (ret) { 2046 drm_dbg_core(dev, 2047 "crtc %d failed to acquire vblank counter, %d\n", 2048 pipe, ret); 2049 return ret; 2050 } 2051 } 2052 drm_modeset_lock(&crtc->mutex, NULL); 2053 if (crtc->state) 2054 get_seq->active = crtc->state->enable; 2055 else 2056 get_seq->active = crtc->enabled; 2057 drm_modeset_unlock(&crtc->mutex); 2058 get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now); 2059 get_seq->sequence_ns = ktime_to_ns(now); 2060 if (!vblank_enabled) 2061 drm_crtc_vblank_put(crtc); 2062 return 0; 2063} 2064 2065/* 2066 * Queue a event for VBLANK sequence 2067 * 2068 * \param dev DRM device 2069 * \param data user argument, pointing to a drm_crtc_queue_sequence structure. 2070 * \param file_priv drm file private for the user's open file descriptor 2071 */ 2072 2073int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data, 2074 struct drm_file *file_priv) 2075{ 2076 struct drm_crtc *crtc; 2077 struct drm_vblank_crtc *vblank; 2078 int pipe; 2079 struct drm_crtc_queue_sequence *queue_seq = data; 2080 ktime_t now; 2081 struct drm_pending_vblank_event *e; 2082 u32 flags; 2083 u64 seq; 2084 u64 req_seq; 2085 int ret; 2086 2087 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 2088 return -EOPNOTSUPP; 2089 2090 if (!drm_dev_has_vblank(dev)) 2091 return -EOPNOTSUPP; 2092 2093 crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id); 2094 if (!crtc) 2095 return -ENOENT; 2096 2097 flags = queue_seq->flags; 2098 /* Check valid flag bits */ 2099 if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE| 2100 DRM_CRTC_SEQUENCE_NEXT_ON_MISS)) 2101 return -EINVAL; 2102 2103 pipe = drm_crtc_index(crtc); 2104 2105 vblank = drm_crtc_vblank_crtc(crtc); 2106 2107 e = kzalloc(sizeof(*e), GFP_KERNEL); 2108 if (e == NULL) 2109 return -ENOMEM; 2110 2111 ret = drm_crtc_vblank_get(crtc); 2112 if (ret) { 2113 drm_dbg_core(dev, 2114 "crtc %d failed to acquire vblank counter, %d\n", 2115 pipe, ret); 2116 goto err_free; 2117 } 2118 2119 seq = drm_vblank_count_and_time(dev, pipe, &now); 2120 req_seq = queue_seq->sequence; 2121 2122 if (flags & DRM_CRTC_SEQUENCE_RELATIVE) 2123 req_seq += seq; 2124 2125 if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && drm_vblank_passed(seq, req_seq)) 2126 req_seq = seq + 1; 2127 2128 e->pipe = pipe; 2129 e->event.base.type = DRM_EVENT_CRTC_SEQUENCE; 2130 e->event.base.length = sizeof(e->event.seq); 2131 e->event.seq.user_data = queue_seq->user_data; 2132 2133 spin_lock_irq(&dev->event_lock); 2134 2135 /* 2136 * drm_crtc_vblank_off() might have been called after we called 2137 * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the 2138 * vblank disable, so no need for further locking. The reference from 2139 * drm_crtc_vblank_get() protects against vblank disable from another source. 2140 */ 2141 if (!READ_ONCE(vblank->enabled)) { 2142 ret = -EINVAL; 2143 goto err_unlock; 2144 } 2145 2146 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base, 2147 &e->event.base); 2148 2149 if (ret) 2150 goto err_unlock; 2151 2152 e->sequence = req_seq; 2153 2154 if (drm_vblank_passed(seq, req_seq)) { 2155 drm_crtc_vblank_put(crtc); 2156 send_vblank_event(dev, e, seq, now); 2157 queue_seq->sequence = seq; 2158 } else { 2159 /* drm_handle_vblank_events will call drm_vblank_put */ 2160 list_add_tail(&e->base.link, &dev->vblank_event_list); 2161 queue_seq->sequence = req_seq; 2162 } 2163 2164 spin_unlock_irq(&dev->event_lock); 2165 return 0; 2166 2167err_unlock: 2168 spin_unlock_irq(&dev->event_lock); 2169 drm_crtc_vblank_put(crtc); 2170err_free: 2171 kfree(e); 2172 return ret; 2173} 2174 2175/* 2176 * VBLANK timer 2177 */ 2178 2179static enum hrtimer_restart drm_vblank_timer_function(struct hrtimer *timer) 2180{ 2181 struct drm_vblank_crtc_timer *vtimer = 2182 container_of(timer, struct drm_vblank_crtc_timer, timer); 2183 struct drm_crtc *crtc = vtimer->crtc; 2184 const struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private; 2185 struct drm_device *dev = crtc->dev; 2186 unsigned long flags; 2187 ktime_t interval; 2188 u64 ret_overrun; 2189 bool succ; 2190 2191 spin_lock_irqsave(&vtimer->interval_lock, flags); 2192 interval = vtimer->interval; 2193 spin_unlock_irqrestore(&vtimer->interval_lock, flags); 2194 2195 if (!interval) 2196 return HRTIMER_NORESTART; 2197 2198 ret_overrun = hrtimer_forward_now(&vtimer->timer, interval); 2199 if (ret_overrun != 1) 2200 drm_dbg_vbl(dev, "vblank timer overrun\n"); 2201 2202 if (crtc_funcs->handle_vblank_timeout) 2203 succ = crtc_funcs->handle_vblank_timeout(crtc); 2204 else 2205 succ = drm_crtc_handle_vblank(crtc); 2206 if (!succ) 2207 return HRTIMER_NORESTART; 2208 2209 return HRTIMER_RESTART; 2210} 2211 2212/** 2213 * drm_crtc_vblank_start_timer - Starts the vblank timer on the given CRTC 2214 * @crtc: the CRTC 2215 * 2216 * Drivers should call this function from their CRTC's enable_vblank 2217 * function to start a vblank timer. The timer will fire after the duration 2218 * of a full frame. drm_crtc_vblank_cancel_timer() disables a running timer. 2219 * 2220 * Returns: 2221 * 0 on success, or a negative errno code otherwise. 2222 */ 2223int drm_crtc_vblank_start_timer(struct drm_crtc *crtc) 2224{ 2225 struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc); 2226 struct drm_vblank_crtc_timer *vtimer = &vblank->vblank_timer; 2227 unsigned long flags; 2228 2229 if (!vtimer->crtc) { 2230 /* 2231 * Set up the data structures on the first invocation. 2232 */ 2233 vtimer->crtc = crtc; 2234 spin_lock_init(&vtimer->interval_lock); 2235 hrtimer_setup(&vtimer->timer, drm_vblank_timer_function, 2236 CLOCK_MONOTONIC, HRTIMER_MODE_REL); 2237 } else { 2238 /* 2239 * Timer should not be active. If it is, wait for the 2240 * previous cancel operations to finish. 2241 */ 2242 while (hrtimer_active(&vtimer->timer)) 2243 hrtimer_try_to_cancel(&vtimer->timer); 2244 } 2245 2246 drm_calc_timestamping_constants(crtc, &crtc->mode); 2247 2248 spin_lock_irqsave(&vtimer->interval_lock, flags); 2249 vtimer->interval = ns_to_ktime(vblank->framedur_ns); 2250 spin_unlock_irqrestore(&vtimer->interval_lock, flags); 2251 2252 hrtimer_start(&vtimer->timer, vtimer->interval, HRTIMER_MODE_REL); 2253 2254 return 0; 2255} 2256EXPORT_SYMBOL(drm_crtc_vblank_start_timer); 2257 2258/** 2259 * drm_crtc_vblank_cancel_timer - Cancels the given CRTC's vblank timer 2260 * @crtc: the CRTC 2261 * 2262 * Drivers should call this function from their CRTC's disable_vblank 2263 * function to stop a vblank timer. 2264 */ 2265void drm_crtc_vblank_cancel_timer(struct drm_crtc *crtc) 2266{ 2267 struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc); 2268 struct drm_vblank_crtc_timer *vtimer = &vblank->vblank_timer; 2269 unsigned long flags; 2270 2271 /* 2272 * Calling hrtimer_cancel() can result in a deadlock with DRM's 2273 * vblank_time_lime_lock and hrtimers' softirq_expiry_lock. So 2274 * clear interval and indicate cancellation. The timer function 2275 * will cancel itself on the next invocation. 2276 */ 2277 2278 spin_lock_irqsave(&vtimer->interval_lock, flags); 2279 vtimer->interval = 0; 2280 spin_unlock_irqrestore(&vtimer->interval_lock, flags); 2281 2282 hrtimer_try_to_cancel(&vtimer->timer); 2283} 2284EXPORT_SYMBOL(drm_crtc_vblank_cancel_timer); 2285 2286/** 2287 * drm_crtc_vblank_get_vblank_timeout - Returns the vblank timeout 2288 * @crtc: The CRTC 2289 * @vblank_time: Returns the next vblank timestamp 2290 * 2291 * The helper drm_crtc_vblank_get_vblank_timeout() returns the next vblank 2292 * timestamp of the CRTC's vblank timer according to the timer's expiry 2293 * time. 2294 */ 2295void drm_crtc_vblank_get_vblank_timeout(struct drm_crtc *crtc, ktime_t *vblank_time) 2296{ 2297 struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc); 2298 struct drm_vblank_crtc_timer *vtimer = &vblank->vblank_timer; 2299 u64 cur_count; 2300 ktime_t cur_time; 2301 2302 if (!READ_ONCE(vblank->enabled)) { 2303 *vblank_time = ktime_get(); 2304 return; 2305 } 2306 2307 /* 2308 * A concurrent vblank timeout could update the expires field before 2309 * we compare it with the vblank time. Hence we'd compare the old 2310 * expiry time to the new vblank time; deducing the timer had already 2311 * expired. Reread until we get consistent values from both fields. 2312 */ 2313 do { 2314 cur_count = drm_crtc_vblank_count_and_time(crtc, &cur_time); 2315 *vblank_time = READ_ONCE(vtimer->timer.node.expires); 2316 } while (cur_count != drm_crtc_vblank_count_and_time(crtc, &cur_time)); 2317 2318 if (drm_WARN_ON(crtc->dev, !ktime_compare(*vblank_time, cur_time))) 2319 return; /* Already expired */ 2320 2321 /* 2322 * To prevent races we roll the hrtimer forward before we do any 2323 * interrupt processing - this is how real hw works (the interrupt 2324 * is only generated after all the vblank registers are updated) 2325 * and what the vblank core expects. Therefore we need to always 2326 * correct the timestamp by one frame. 2327 */ 2328 *vblank_time = ktime_sub(*vblank_time, vtimer->interval); 2329} 2330EXPORT_SYMBOL(drm_crtc_vblank_get_vblank_timeout);
~/
Read-only file preview.