FILE
panthor_sched.c
File preview with syntax highlighting.
linux-6.19.14/drivers/gpu/drm/panthor/panthor_sched.c
1// SPDX-License-Identifier: GPL-2.0 or MIT 2/* Copyright 2023 Collabora ltd. */ 3 4#include <drm/drm_drv.h> 5#include <drm/drm_exec.h> 6#include <drm/drm_gem_shmem_helper.h> 7#include <drm/drm_managed.h> 8#include <drm/drm_print.h> 9#include <drm/gpu_scheduler.h> 10#include <drm/panthor_drm.h> 11 12#include <linux/build_bug.h> 13#include <linux/cleanup.h> 14#include <linux/clk.h> 15#include <linux/delay.h> 16#include <linux/dma-mapping.h> 17#include <linux/dma-resv.h> 18#include <linux/firmware.h> 19#include <linux/interrupt.h> 20#include <linux/io.h> 21#include <linux/iopoll.h> 22#include <linux/iosys-map.h> 23#include <linux/module.h> 24#include <linux/platform_device.h> 25#include <linux/pm_runtime.h> 26#include <linux/rcupdate.h> 27 28#include "panthor_devfreq.h" 29#include "panthor_device.h" 30#include "panthor_fw.h" 31#include "panthor_gem.h" 32#include "panthor_gpu.h" 33#include "panthor_heap.h" 34#include "panthor_mmu.h" 35#include "panthor_regs.h" 36#include "panthor_sched.h" 37 38/** 39 * DOC: Scheduler 40 * 41 * Mali CSF hardware adopts a firmware-assisted scheduling model, where 42 * the firmware takes care of scheduling aspects, to some extent. 43 * 44 * The scheduling happens at the scheduling group level, each group 45 * contains 1 to N queues (N is FW/hardware dependent, and exposed 46 * through the firmware interface). Each queue is assigned a command 47 * stream ring buffer, which serves as a way to get jobs submitted to 48 * the GPU, among other things. 49 * 50 * The firmware can schedule a maximum of M groups (M is FW/hardware 51 * dependent, and exposed through the firmware interface). Passed 52 * this maximum number of groups, the kernel must take care of 53 * rotating the groups passed to the firmware so every group gets 54 * a chance to have his queues scheduled for execution. 55 * 56 * The current implementation only supports with kernel-mode queues. 57 * In other terms, userspace doesn't have access to the ring-buffer. 58 * Instead, userspace passes indirect command stream buffers that are 59 * called from the queue ring-buffer by the kernel using a pre-defined 60 * sequence of command stream instructions to ensure the userspace driver 61 * always gets consistent results (cache maintenance, 62 * synchronization, ...). 63 * 64 * We rely on the drm_gpu_scheduler framework to deal with job 65 * dependencies and submission. As any other driver dealing with a 66 * FW-scheduler, we use the 1:1 entity:scheduler mode, such that each 67 * entity has its own job scheduler. When a job is ready to be executed 68 * (all its dependencies are met), it is pushed to the appropriate 69 * queue ring-buffer, and the group is scheduled for execution if it 70 * wasn't already active. 71 * 72 * Kernel-side group scheduling is timeslice-based. When we have less 73 * groups than there are slots, the periodic tick is disabled and we 74 * just let the FW schedule the active groups. When there are more 75 * groups than slots, we let each group a chance to execute stuff for 76 * a given amount of time, and then re-evaluate and pick new groups 77 * to schedule. The group selection algorithm is based on 78 * priority+round-robin. 79 * 80 * Even though user-mode queues is out of the scope right now, the 81 * current design takes them into account by avoiding any guess on the 82 * group/queue state that would be based on information we wouldn't have 83 * if userspace was in charge of the ring-buffer. That's also one of the 84 * reason we don't do 'cooperative' scheduling (encoding FW group slot 85 * reservation as dma_fence that would be returned from the 86 * drm_gpu_scheduler::prepare_job() hook, and treating group rotation as 87 * a queue of waiters, ordered by job submission order). This approach 88 * would work for kernel-mode queues, but would make user-mode queues a 89 * lot more complicated to retrofit. 90 */ 91 92#define JOB_TIMEOUT_MS 5000 93 94#define MAX_CSG_PRIO 0xf 95 96#define NUM_INSTRS_PER_CACHE_LINE (64 / sizeof(u64)) 97#define MAX_INSTRS_PER_JOB 24 98 99struct panthor_group; 100 101/** 102 * struct panthor_csg_slot - Command stream group slot 103 * 104 * This represents a FW slot for a scheduling group. 105 */ 106struct panthor_csg_slot { 107 /** @group: Scheduling group bound to this slot. */ 108 struct panthor_group *group; 109 110 /** @priority: Group priority. */ 111 u8 priority; 112 113 /** 114 * @idle: True if the group bound to this slot is idle. 115 * 116 * A group is idle when it has nothing waiting for execution on 117 * all its queues, or when queues are blocked waiting for something 118 * to happen (synchronization object). 119 */ 120 bool idle; 121}; 122 123/** 124 * enum panthor_csg_priority - Group priority 125 */ 126enum panthor_csg_priority { 127 /** @PANTHOR_CSG_PRIORITY_LOW: Low priority group. */ 128 PANTHOR_CSG_PRIORITY_LOW = 0, 129 130 /** @PANTHOR_CSG_PRIORITY_MEDIUM: Medium priority group. */ 131 PANTHOR_CSG_PRIORITY_MEDIUM, 132 133 /** @PANTHOR_CSG_PRIORITY_HIGH: High priority group. */ 134 PANTHOR_CSG_PRIORITY_HIGH, 135 136 /** 137 * @PANTHOR_CSG_PRIORITY_RT: Real-time priority group. 138 * 139 * Real-time priority allows one to preempt scheduling of other 140 * non-real-time groups. When such a group becomes executable, 141 * it will evict the group with the lowest non-rt priority if 142 * there's no free group slot available. 143 */ 144 PANTHOR_CSG_PRIORITY_RT, 145 146 /** @PANTHOR_CSG_PRIORITY_COUNT: Number of priority levels. */ 147 PANTHOR_CSG_PRIORITY_COUNT, 148}; 149 150/** 151 * struct panthor_scheduler - Object used to manage the scheduler 152 */ 153struct panthor_scheduler { 154 /** @ptdev: Device. */ 155 struct panthor_device *ptdev; 156 157 /** 158 * @wq: Workqueue used by our internal scheduler logic and 159 * drm_gpu_scheduler. 160 * 161 * Used for the scheduler tick, group update or other kind of FW 162 * event processing that can't be handled in the threaded interrupt 163 * path. Also passed to the drm_gpu_scheduler instances embedded 164 * in panthor_queue. 165 */ 166 struct workqueue_struct *wq; 167 168 /** 169 * @heap_alloc_wq: Workqueue used to schedule tiler_oom works. 170 * 171 * We have a queue dedicated to heap chunk allocation works to avoid 172 * blocking the rest of the scheduler if the allocation tries to 173 * reclaim memory. 174 */ 175 struct workqueue_struct *heap_alloc_wq; 176 177 /** @tick_work: Work executed on a scheduling tick. */ 178 struct delayed_work tick_work; 179 180 /** 181 * @sync_upd_work: Work used to process synchronization object updates. 182 * 183 * We use this work to unblock queues/groups that were waiting on a 184 * synchronization object. 185 */ 186 struct work_struct sync_upd_work; 187 188 /** 189 * @fw_events_work: Work used to process FW events outside the interrupt path. 190 * 191 * Even if the interrupt is threaded, we need any event processing 192 * that require taking the panthor_scheduler::lock to be processed 193 * outside the interrupt path so we don't block the tick logic when 194 * it calls panthor_fw_{csg,wait}_wait_acks(). Since most of the 195 * event processing requires taking this lock, we just delegate all 196 * FW event processing to the scheduler workqueue. 197 */ 198 struct work_struct fw_events_work; 199 200 /** 201 * @fw_events: Bitmask encoding pending FW events. 202 */ 203 atomic_t fw_events; 204 205 /** 206 * @resched_target: When the next tick should occur. 207 * 208 * Expressed in jiffies. 209 */ 210 u64 resched_target; 211 212 /** 213 * @last_tick: When the last tick occurred. 214 * 215 * Expressed in jiffies. 216 */ 217 u64 last_tick; 218 219 /** @tick_period: Tick period in jiffies. */ 220 u64 tick_period; 221 222 /** 223 * @lock: Lock protecting access to all the scheduler fields. 224 * 225 * Should be taken in the tick work, the irq handler, and anywhere the @groups 226 * fields are touched. 227 */ 228 struct mutex lock; 229 230 /** @groups: Various lists used to classify groups. */ 231 struct { 232 /** 233 * @runnable: Runnable group lists. 234 * 235 * When a group has queues that want to execute something, 236 * its panthor_group::run_node should be inserted here. 237 * 238 * One list per-priority. 239 */ 240 struct list_head runnable[PANTHOR_CSG_PRIORITY_COUNT]; 241 242 /** 243 * @idle: Idle group lists. 244 * 245 * When all queues of a group are idle (either because they 246 * have nothing to execute, or because they are blocked), the 247 * panthor_group::run_node field should be inserted here. 248 * 249 * One list per-priority. 250 */ 251 struct list_head idle[PANTHOR_CSG_PRIORITY_COUNT]; 252 253 /** 254 * @waiting: List of groups whose queues are blocked on a 255 * synchronization object. 256 * 257 * Insert panthor_group::wait_node here when a group is waiting 258 * for synchronization objects to be signaled. 259 * 260 * This list is evaluated in the @sync_upd_work work. 261 */ 262 struct list_head waiting; 263 } groups; 264 265 /** 266 * @csg_slots: FW command stream group slots. 267 */ 268 struct panthor_csg_slot csg_slots[MAX_CSGS]; 269 270 /** @csg_slot_count: Number of command stream group slots exposed by the FW. */ 271 u32 csg_slot_count; 272 273 /** @cs_slot_count: Number of command stream slot per group slot exposed by the FW. */ 274 u32 cs_slot_count; 275 276 /** @as_slot_count: Number of address space slots supported by the MMU. */ 277 u32 as_slot_count; 278 279 /** @used_csg_slot_count: Number of command stream group slot currently used. */ 280 u32 used_csg_slot_count; 281 282 /** @sb_slot_count: Number of scoreboard slots. */ 283 u32 sb_slot_count; 284 285 /** 286 * @might_have_idle_groups: True if an active group might have become idle. 287 * 288 * This will force a tick, so other runnable groups can be scheduled if one 289 * or more active groups became idle. 290 */ 291 bool might_have_idle_groups; 292 293 /** @pm: Power management related fields. */ 294 struct { 295 /** @has_ref: True if the scheduler owns a runtime PM reference. */ 296 bool has_ref; 297 } pm; 298 299 /** @reset: Reset related fields. */ 300 struct { 301 /** @lock: Lock protecting the other reset fields. */ 302 struct mutex lock; 303 304 /** 305 * @in_progress: True if a reset is in progress. 306 * 307 * Set to true in panthor_sched_pre_reset() and back to false in 308 * panthor_sched_post_reset(). 309 */ 310 atomic_t in_progress; 311 312 /** 313 * @stopped_groups: List containing all groups that were stopped 314 * before a reset. 315 * 316 * Insert panthor_group::run_node in the pre_reset path. 317 */ 318 struct list_head stopped_groups; 319 } reset; 320}; 321 322/** 323 * struct panthor_syncobj_32b - 32-bit FW synchronization object 324 */ 325struct panthor_syncobj_32b { 326 /** @seqno: Sequence number. */ 327 u32 seqno; 328 329 /** 330 * @status: Status. 331 * 332 * Not zero on failure. 333 */ 334 u32 status; 335}; 336 337/** 338 * struct panthor_syncobj_64b - 64-bit FW synchronization object 339 */ 340struct panthor_syncobj_64b { 341 /** @seqno: Sequence number. */ 342 u64 seqno; 343 344 /** 345 * @status: Status. 346 * 347 * Not zero on failure. 348 */ 349 u32 status; 350 351 /** @pad: MBZ. */ 352 u32 pad; 353}; 354 355/** 356 * struct panthor_queue - Execution queue 357 */ 358struct panthor_queue { 359 /** @scheduler: DRM scheduler used for this queue. */ 360 struct drm_gpu_scheduler scheduler; 361 362 /** @entity: DRM scheduling entity used for this queue. */ 363 struct drm_sched_entity entity; 364 365 /** @name: DRM scheduler name for this queue. */ 366 char *name; 367 368 /** @timeout: Queue timeout related fields. */ 369 struct { 370 /** @timeout.work: Work executed when a queue timeout occurs. */ 371 struct delayed_work work; 372 373 /** 374 * @timeout.remaining: Time remaining before a queue timeout. 375 * 376 * When the timer is running, this value is set to MAX_SCHEDULE_TIMEOUT. 377 * When the timer is suspended, it's set to the time remaining when the 378 * timer was suspended. 379 */ 380 unsigned long remaining; 381 } timeout; 382 383 /** 384 * @doorbell_id: Doorbell assigned to this queue. 385 * 386 * Right now, all groups share the same doorbell, and the doorbell ID 387 * is assigned to group_slot + 1 when the group is assigned a slot. But 388 * we might decide to provide fine grained doorbell assignment at some 389 * point, so don't have to wake up all queues in a group every time one 390 * of them is updated. 391 */ 392 u8 doorbell_id; 393 394 /** 395 * @priority: Priority of the queue inside the group. 396 * 397 * Must be less than 16 (Only 4 bits available). 398 */ 399 u8 priority; 400#define CSF_MAX_QUEUE_PRIO GENMASK(3, 0) 401 402 /** @ringbuf: Command stream ring-buffer. */ 403 struct panthor_kernel_bo *ringbuf; 404 405 /** @iface: Firmware interface. */ 406 struct { 407 /** @mem: FW memory allocated for this interface. */ 408 struct panthor_kernel_bo *mem; 409 410 /** @input: Input interface. */ 411 struct panthor_fw_ringbuf_input_iface *input; 412 413 /** @output: Output interface. */ 414 const struct panthor_fw_ringbuf_output_iface *output; 415 416 /** @input_fw_va: FW virtual address of the input interface buffer. */ 417 u32 input_fw_va; 418 419 /** @output_fw_va: FW virtual address of the output interface buffer. */ 420 u32 output_fw_va; 421 } iface; 422 423 /** 424 * @syncwait: Stores information about the synchronization object this 425 * queue is waiting on. 426 */ 427 struct { 428 /** @gpu_va: GPU address of the synchronization object. */ 429 u64 gpu_va; 430 431 /** @ref: Reference value to compare against. */ 432 u64 ref; 433 434 /** @gt: True if this is a greater-than test. */ 435 bool gt; 436 437 /** @sync64: True if this is a 64-bit sync object. */ 438 bool sync64; 439 440 /** @bo: Buffer object holding the synchronization object. */ 441 struct drm_gem_object *obj; 442 443 /** @offset: Offset of the synchronization object inside @bo. */ 444 u64 offset; 445 446 /** 447 * @kmap: Kernel mapping of the buffer object holding the 448 * synchronization object. 449 */ 450 void *kmap; 451 } syncwait; 452 453 /** @fence_ctx: Fence context fields. */ 454 struct { 455 /** @lock: Used to protect access to all fences allocated by this context. */ 456 spinlock_t lock; 457 458 /** 459 * @id: Fence context ID. 460 * 461 * Allocated with dma_fence_context_alloc(). 462 */ 463 u64 id; 464 465 /** @seqno: Sequence number of the last initialized fence. */ 466 atomic64_t seqno; 467 468 /** 469 * @last_fence: Fence of the last submitted job. 470 * 471 * We return this fence when we get an empty command stream. 472 * This way, we are guaranteed that all earlier jobs have completed 473 * when drm_sched_job::s_fence::finished without having to feed 474 * the CS ring buffer with a dummy job that only signals the fence. 475 */ 476 struct dma_fence *last_fence; 477 478 /** 479 * @in_flight_jobs: List containing all in-flight jobs. 480 * 481 * Used to keep track and signal panthor_job::done_fence when the 482 * synchronization object attached to the queue is signaled. 483 */ 484 struct list_head in_flight_jobs; 485 } fence_ctx; 486 487 /** @profiling: Job profiling data slots and access information. */ 488 struct { 489 /** @slots: Kernel BO holding the slots. */ 490 struct panthor_kernel_bo *slots; 491 492 /** @slot_count: Number of jobs ringbuffer can hold at once. */ 493 u32 slot_count; 494 495 /** @seqno: Index of the next available profiling information slot. */ 496 u32 seqno; 497 } profiling; 498}; 499 500/** 501 * enum panthor_group_state - Scheduling group state. 502 */ 503enum panthor_group_state { 504 /** @PANTHOR_CS_GROUP_CREATED: Group was created, but not scheduled yet. */ 505 PANTHOR_CS_GROUP_CREATED, 506 507 /** @PANTHOR_CS_GROUP_ACTIVE: Group is currently scheduled. */ 508 PANTHOR_CS_GROUP_ACTIVE, 509 510 /** 511 * @PANTHOR_CS_GROUP_SUSPENDED: Group was scheduled at least once, but is 512 * inactive/suspended right now. 513 */ 514 PANTHOR_CS_GROUP_SUSPENDED, 515 516 /** 517 * @PANTHOR_CS_GROUP_TERMINATED: Group was terminated. 518 * 519 * Can no longer be scheduled. The only allowed action is a destruction. 520 */ 521 PANTHOR_CS_GROUP_TERMINATED, 522 523 /** 524 * @PANTHOR_CS_GROUP_UNKNOWN_STATE: Group is an unknown state. 525 * 526 * The FW returned an inconsistent state. The group is flagged unusable 527 * and can no longer be scheduled. The only allowed action is a 528 * destruction. 529 * 530 * When that happens, we also schedule a FW reset, to start from a fresh 531 * state. 532 */ 533 PANTHOR_CS_GROUP_UNKNOWN_STATE, 534}; 535 536/** 537 * struct panthor_group - Scheduling group object 538 */ 539struct panthor_group { 540 /** @refcount: Reference count */ 541 struct kref refcount; 542 543 /** @ptdev: Device. */ 544 struct panthor_device *ptdev; 545 546 /** @vm: VM bound to the group. */ 547 struct panthor_vm *vm; 548 549 /** @compute_core_mask: Mask of shader cores that can be used for compute jobs. */ 550 u64 compute_core_mask; 551 552 /** @fragment_core_mask: Mask of shader cores that can be used for fragment jobs. */ 553 u64 fragment_core_mask; 554 555 /** @tiler_core_mask: Mask of tiler cores that can be used for tiler jobs. */ 556 u64 tiler_core_mask; 557 558 /** @max_compute_cores: Maximum number of shader cores used for compute jobs. */ 559 u8 max_compute_cores; 560 561 /** @max_fragment_cores: Maximum number of shader cores used for fragment jobs. */ 562 u8 max_fragment_cores; 563 564 /** @max_tiler_cores: Maximum number of tiler cores used for tiler jobs. */ 565 u8 max_tiler_cores; 566 567 /** @priority: Group priority (check panthor_csg_priority). */ 568 u8 priority; 569 570 /** @blocked_queues: Bitmask reflecting the blocked queues. */ 571 u32 blocked_queues; 572 573 /** @idle_queues: Bitmask reflecting the idle queues. */ 574 u32 idle_queues; 575 576 /** @fatal_lock: Lock used to protect access to fatal fields. */ 577 spinlock_t fatal_lock; 578 579 /** @fatal_queues: Bitmask reflecting the queues that hit a fatal exception. */ 580 u32 fatal_queues; 581 582 /** @tiler_oom: Mask of queues that have a tiler OOM event to process. */ 583 atomic_t tiler_oom; 584 585 /** @queue_count: Number of queues in this group. */ 586 u32 queue_count; 587 588 /** @queues: Queues owned by this group. */ 589 struct panthor_queue *queues[MAX_CS_PER_CSG]; 590 591 /** 592 * @csg_id: ID of the FW group slot. 593 * 594 * -1 when the group is not scheduled/active. 595 */ 596 int csg_id; 597 598 /** 599 * @destroyed: True when the group has been destroyed. 600 * 601 * If a group is destroyed it becomes useless: no further jobs can be submitted 602 * to its queues. We simply wait for all references to be dropped so we can 603 * release the group object. 604 */ 605 bool destroyed; 606 607 /** 608 * @timedout: True when a timeout occurred on any of the queues owned by 609 * this group. 610 * 611 * Timeouts can be reported by drm_sched or by the FW. If a reset is required, 612 * and the group can't be suspended, this also leads to a timeout. In any case, 613 * any timeout situation is unrecoverable, and the group becomes useless. We 614 * simply wait for all references to be dropped so we can release the group 615 * object. 616 */ 617 bool timedout; 618 619 /** 620 * @innocent: True when the group becomes unusable because the group suspension 621 * failed during a reset. 622 * 623 * Sometimes the FW was put in a bad state by other groups, causing the group 624 * suspension happening in the reset path to fail. In that case, we consider the 625 * group innocent. 626 */ 627 bool innocent; 628 629 /** 630 * @syncobjs: Pool of per-queue synchronization objects. 631 * 632 * One sync object per queue. The position of the sync object is 633 * determined by the queue index. 634 */ 635 struct panthor_kernel_bo *syncobjs; 636 637 /** @fdinfo: Per-file info exposed through /proc/<process>/fdinfo */ 638 struct { 639 /** @data: Total sampled values for jobs in queues from this group. */ 640 struct panthor_gpu_usage data; 641 642 /** 643 * @fdinfo.lock: Spinlock to govern concurrent access from drm file's fdinfo 644 * callback and job post-completion processing function 645 */ 646 spinlock_t lock; 647 648 /** @fdinfo.kbo_sizes: Aggregate size of private kernel BO's held by the group. */ 649 size_t kbo_sizes; 650 } fdinfo; 651 652 /** @task_info: Info of current->group_leader that created the group. */ 653 struct { 654 /** @task_info.pid: pid of current->group_leader */ 655 pid_t pid; 656 657 /** @task_info.comm: comm of current->group_leader */ 658 char comm[TASK_COMM_LEN]; 659 } task_info; 660 661 /** @state: Group state. */ 662 enum panthor_group_state state; 663 664 /** 665 * @suspend_buf: Suspend buffer. 666 * 667 * Stores the state of the group and its queues when a group is suspended. 668 * Used at resume time to restore the group in its previous state. 669 * 670 * The size of the suspend buffer is exposed through the FW interface. 671 */ 672 struct panthor_kernel_bo *suspend_buf; 673 674 /** 675 * @protm_suspend_buf: Protection mode suspend buffer. 676 * 677 * Stores the state of the group and its queues when a group that's in 678 * protection mode is suspended. 679 * 680 * Used at resume time to restore the group in its previous state. 681 * 682 * The size of the protection mode suspend buffer is exposed through the 683 * FW interface. 684 */ 685 struct panthor_kernel_bo *protm_suspend_buf; 686 687 /** @sync_upd_work: Work used to check/signal job fences. */ 688 struct work_struct sync_upd_work; 689 690 /** @tiler_oom_work: Work used to process tiler OOM events happening on this group. */ 691 struct work_struct tiler_oom_work; 692 693 /** @term_work: Work used to finish the group termination procedure. */ 694 struct work_struct term_work; 695 696 /** 697 * @release_work: Work used to release group resources. 698 * 699 * We need to postpone the group release to avoid a deadlock when 700 * the last ref is released in the tick work. 701 */ 702 struct work_struct release_work; 703 704 /** 705 * @run_node: Node used to insert the group in the 706 * panthor_group::groups::{runnable,idle} and 707 * panthor_group::reset.stopped_groups lists. 708 */ 709 struct list_head run_node; 710 711 /** 712 * @wait_node: Node used to insert the group in the 713 * panthor_group::groups::waiting list. 714 */ 715 struct list_head wait_node; 716}; 717 718struct panthor_job_profiling_data { 719 struct { 720 u64 before; 721 u64 after; 722 } cycles; 723 724 struct { 725 u64 before; 726 u64 after; 727 } time; 728}; 729 730/** 731 * group_queue_work() - Queue a group work 732 * @group: Group to queue the work for. 733 * @wname: Work name. 734 * 735 * Grabs a ref and queue a work item to the scheduler workqueue. If 736 * the work was already queued, we release the reference we grabbed. 737 * 738 * Work callbacks must release the reference we grabbed here. 739 */ 740#define group_queue_work(group, wname) \ 741 do { \ 742 group_get(group); \ 743 if (!queue_work((group)->ptdev->scheduler->wq, &(group)->wname ## _work)) \ 744 group_put(group); \ 745 } while (0) 746 747/** 748 * sched_queue_work() - Queue a scheduler work. 749 * @sched: Scheduler object. 750 * @wname: Work name. 751 * 752 * Conditionally queues a scheduler work if no reset is pending/in-progress. 753 */ 754#define sched_queue_work(sched, wname) \ 755 do { \ 756 if (!atomic_read(&(sched)->reset.in_progress) && \ 757 !panthor_device_reset_is_pending((sched)->ptdev)) \ 758 queue_work((sched)->wq, &(sched)->wname ## _work); \ 759 } while (0) 760 761/** 762 * sched_queue_delayed_work() - Queue a scheduler delayed work. 763 * @sched: Scheduler object. 764 * @wname: Work name. 765 * @delay: Work delay in jiffies. 766 * 767 * Conditionally queues a scheduler delayed work if no reset is 768 * pending/in-progress. 769 */ 770#define sched_queue_delayed_work(sched, wname, delay) \ 771 do { \ 772 if (!atomic_read(&sched->reset.in_progress) && \ 773 !panthor_device_reset_is_pending((sched)->ptdev)) \ 774 mod_delayed_work((sched)->wq, &(sched)->wname ## _work, delay); \ 775 } while (0) 776 777/* 778 * We currently set the maximum of groups per file to an arbitrary low value. 779 * But this can be updated if we need more. 780 */ 781#define MAX_GROUPS_PER_POOL 128 782 783/* 784 * Mark added on an entry of group pool Xarray to identify if the group has 785 * been fully initialized and can be accessed elsewhere in the driver code. 786 */ 787#define GROUP_REGISTERED XA_MARK_1 788 789/** 790 * struct panthor_group_pool - Group pool 791 * 792 * Each file get assigned a group pool. 793 */ 794struct panthor_group_pool { 795 /** @xa: Xarray used to manage group handles. */ 796 struct xarray xa; 797}; 798 799/** 800 * struct panthor_job - Used to manage GPU job 801 */ 802struct panthor_job { 803 /** @base: Inherit from drm_sched_job. */ 804 struct drm_sched_job base; 805 806 /** @refcount: Reference count. */ 807 struct kref refcount; 808 809 /** @group: Group of the queue this job will be pushed to. */ 810 struct panthor_group *group; 811 812 /** @queue_idx: Index of the queue inside @group. */ 813 u32 queue_idx; 814 815 /** @call_info: Information about the userspace command stream call. */ 816 struct { 817 /** @start: GPU address of the userspace command stream. */ 818 u64 start; 819 820 /** @size: Size of the userspace command stream. */ 821 u32 size; 822 823 /** 824 * @latest_flush: Flush ID at the time the userspace command 825 * stream was built. 826 * 827 * Needed for the flush reduction mechanism. 828 */ 829 u32 latest_flush; 830 } call_info; 831 832 /** @ringbuf: Position of this job is in the ring buffer. */ 833 struct { 834 /** @start: Start offset. */ 835 u64 start; 836 837 /** @end: End offset. */ 838 u64 end; 839 } ringbuf; 840 841 /** 842 * @node: Used to insert the job in the panthor_queue::fence_ctx::in_flight_jobs 843 * list. 844 */ 845 struct list_head node; 846 847 /** @done_fence: Fence signaled when the job is finished or cancelled. */ 848 struct dma_fence *done_fence; 849 850 /** @profiling: Job profiling information. */ 851 struct { 852 /** @mask: Current device job profiling enablement bitmask. */ 853 u32 mask; 854 855 /** @slot: Job index in the profiling slots BO. */ 856 u32 slot; 857 } profiling; 858}; 859 860static void 861panthor_queue_put_syncwait_obj(struct panthor_queue *queue) 862{ 863 if (queue->syncwait.kmap) { 864 struct iosys_map map = IOSYS_MAP_INIT_VADDR(queue->syncwait.kmap); 865 866 drm_gem_vunmap(queue->syncwait.obj, &map); 867 queue->syncwait.kmap = NULL; 868 } 869 870 drm_gem_object_put(queue->syncwait.obj); 871 queue->syncwait.obj = NULL; 872} 873 874static void * 875panthor_queue_get_syncwait_obj(struct panthor_group *group, struct panthor_queue *queue) 876{ 877 struct panthor_device *ptdev = group->ptdev; 878 struct panthor_gem_object *bo; 879 struct iosys_map map; 880 int ret; 881 882 if (queue->syncwait.kmap) 883 return queue->syncwait.kmap + queue->syncwait.offset; 884 885 bo = panthor_vm_get_bo_for_va(group->vm, 886 queue->syncwait.gpu_va, 887 &queue->syncwait.offset); 888 if (drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(bo))) 889 goto err_put_syncwait_obj; 890 891 queue->syncwait.obj = &bo->base.base; 892 ret = drm_gem_vmap(queue->syncwait.obj, &map); 893 if (drm_WARN_ON(&ptdev->base, ret)) 894 goto err_put_syncwait_obj; 895 896 queue->syncwait.kmap = map.vaddr; 897 if (drm_WARN_ON(&ptdev->base, !queue->syncwait.kmap)) 898 goto err_put_syncwait_obj; 899 900 return queue->syncwait.kmap + queue->syncwait.offset; 901 902err_put_syncwait_obj: 903 panthor_queue_put_syncwait_obj(queue); 904 return NULL; 905} 906 907static void group_free_queue(struct panthor_group *group, struct panthor_queue *queue) 908{ 909 if (IS_ERR_OR_NULL(queue)) 910 return; 911 912 /* This should have been disabled before that point. */ 913 drm_WARN_ON(&group->ptdev->base, 914 disable_delayed_work_sync(&queue->timeout.work)); 915 916 if (queue->entity.fence_context) 917 drm_sched_entity_destroy(&queue->entity); 918 919 if (queue->scheduler.ops) 920 drm_sched_fini(&queue->scheduler); 921 922 kfree(queue->name); 923 924 panthor_queue_put_syncwait_obj(queue); 925 926 panthor_kernel_bo_destroy(queue->ringbuf); 927 panthor_kernel_bo_destroy(queue->iface.mem); 928 panthor_kernel_bo_destroy(queue->profiling.slots); 929 930 /* Release the last_fence we were holding, if any. */ 931 dma_fence_put(queue->fence_ctx.last_fence); 932 933 kfree(queue); 934} 935 936static void group_release_work(struct work_struct *work) 937{ 938 struct panthor_group *group = container_of(work, 939 struct panthor_group, 940 release_work); 941 u32 i; 942 943 /* dma-fences may still be accessing group->queues under rcu lock. */ 944 synchronize_rcu(); 945 946 for (i = 0; i < group->queue_count; i++) 947 group_free_queue(group, group->queues[i]); 948 949 panthor_kernel_bo_destroy(group->suspend_buf); 950 panthor_kernel_bo_destroy(group->protm_suspend_buf); 951 panthor_kernel_bo_destroy(group->syncobjs); 952 953 panthor_vm_put(group->vm); 954 kfree(group); 955} 956 957static void group_release(struct kref *kref) 958{ 959 struct panthor_group *group = container_of(kref, 960 struct panthor_group, 961 refcount); 962 struct panthor_device *ptdev = group->ptdev; 963 964 drm_WARN_ON(&ptdev->base, group->csg_id >= 0); 965 drm_WARN_ON(&ptdev->base, !list_empty(&group->run_node)); 966 drm_WARN_ON(&ptdev->base, !list_empty(&group->wait_node)); 967 968 queue_work(panthor_cleanup_wq, &group->release_work); 969} 970 971static void group_put(struct panthor_group *group) 972{ 973 if (group) 974 kref_put(&group->refcount, group_release); 975} 976 977static struct panthor_group * 978group_get(struct panthor_group *group) 979{ 980 if (group) 981 kref_get(&group->refcount); 982 983 return group; 984} 985 986/** 987 * group_bind_locked() - Bind a group to a group slot 988 * @group: Group. 989 * @csg_id: Slot. 990 * 991 * Return: 0 on success, a negative error code otherwise. 992 */ 993static int 994group_bind_locked(struct panthor_group *group, u32 csg_id) 995{ 996 struct panthor_device *ptdev = group->ptdev; 997 struct panthor_csg_slot *csg_slot; 998 int ret; 999 1000 lockdep_assert_held(&ptdev->scheduler->lock); 1001 1002 if (drm_WARN_ON(&ptdev->base, group->csg_id != -1 || csg_id >= MAX_CSGS || 1003 ptdev->scheduler->csg_slots[csg_id].group)) 1004 return -EINVAL; 1005 1006 ret = panthor_vm_active(group->vm); 1007 if (ret) 1008 return ret; 1009 1010 csg_slot = &ptdev->scheduler->csg_slots[csg_id]; 1011 group_get(group); 1012 group->csg_id = csg_id; 1013 1014 /* Dummy doorbell allocation: doorbell is assigned to the group and 1015 * all queues use the same doorbell. 1016 * 1017 * TODO: Implement LRU-based doorbell assignment, so the most often 1018 * updated queues get their own doorbell, thus avoiding useless checks 1019 * on queues belonging to the same group that are rarely updated. 1020 */ 1021 for (u32 i = 0; i < group->queue_count; i++) 1022 group->queues[i]->doorbell_id = csg_id + 1; 1023 1024 csg_slot->group = group; 1025 1026 return 0; 1027} 1028 1029/** 1030 * group_unbind_locked() - Unbind a group from a slot. 1031 * @group: Group to unbind. 1032 * 1033 * Return: 0 on success, a negative error code otherwise. 1034 */ 1035static int 1036group_unbind_locked(struct panthor_group *group) 1037{ 1038 struct panthor_device *ptdev = group->ptdev; 1039 struct panthor_csg_slot *slot; 1040 1041 lockdep_assert_held(&ptdev->scheduler->lock); 1042 1043 if (drm_WARN_ON(&ptdev->base, group->csg_id < 0 || group->csg_id >= MAX_CSGS)) 1044 return -EINVAL; 1045 1046 if (drm_WARN_ON(&ptdev->base, group->state == PANTHOR_CS_GROUP_ACTIVE)) 1047 return -EINVAL; 1048 1049 slot = &ptdev->scheduler->csg_slots[group->csg_id]; 1050 panthor_vm_idle(group->vm); 1051 group->csg_id = -1; 1052 1053 /* Tiler OOM events will be re-issued next time the group is scheduled. */ 1054 atomic_set(&group->tiler_oom, 0); 1055 cancel_work(&group->tiler_oom_work); 1056 1057 for (u32 i = 0; i < group->queue_count; i++) 1058 group->queues[i]->doorbell_id = -1; 1059 1060 slot->group = NULL; 1061 1062 group_put(group); 1063 return 0; 1064} 1065 1066static bool 1067group_is_idle(struct panthor_group *group) 1068{ 1069 struct panthor_device *ptdev = group->ptdev; 1070 u32 inactive_queues; 1071 1072 if (group->csg_id >= 0) 1073 return ptdev->scheduler->csg_slots[group->csg_id].idle; 1074 1075 inactive_queues = group->idle_queues | group->blocked_queues; 1076 return hweight32(inactive_queues) == group->queue_count; 1077} 1078 1079static bool 1080group_can_run(struct panthor_group *group) 1081{ 1082 return group->state != PANTHOR_CS_GROUP_TERMINATED && 1083 group->state != PANTHOR_CS_GROUP_UNKNOWN_STATE && 1084 !group->destroyed && group->fatal_queues == 0 && 1085 !group->timedout; 1086} 1087 1088static bool 1089queue_timeout_is_suspended(struct panthor_queue *queue) 1090{ 1091 /* When running, the remaining time is set to MAX_SCHEDULE_TIMEOUT. */ 1092 return queue->timeout.remaining != MAX_SCHEDULE_TIMEOUT; 1093} 1094 1095static void 1096queue_reset_timeout_locked(struct panthor_queue *queue) 1097{ 1098 lockdep_assert_held(&queue->fence_ctx.lock); 1099 1100 if (!queue_timeout_is_suspended(queue)) { 1101 mod_delayed_work(queue->scheduler.timeout_wq, 1102 &queue->timeout.work, 1103 msecs_to_jiffies(JOB_TIMEOUT_MS)); 1104 } 1105} 1106 1107static void 1108queue_suspend_timeout_locked(struct panthor_queue *queue) 1109{ 1110 unsigned long qtimeout, now; 1111 struct panthor_group *group; 1112 struct panthor_job *job; 1113 bool timer_was_active; 1114 1115 lockdep_assert_held(&queue->fence_ctx.lock); 1116 1117 /* Already suspended, nothing to do. */ 1118 if (queue_timeout_is_suspended(queue)) 1119 return; 1120 1121 job = list_first_entry_or_null(&queue->fence_ctx.in_flight_jobs, 1122 struct panthor_job, node); 1123 group = job ? job->group : NULL; 1124 1125 /* If the queue is blocked and the group is idle, we want the timer to 1126 * keep running because the group can't be unblocked by other queues, 1127 * so it has to come from an external source, and we want to timebox 1128 * this external signalling. 1129 */ 1130 if (group && group_can_run(group) && 1131 (group->blocked_queues & BIT(job->queue_idx)) && 1132 group_is_idle(group)) 1133 return; 1134 1135 now = jiffies; 1136 qtimeout = queue->timeout.work.timer.expires; 1137 1138 /* Cancel the timer. */ 1139 timer_was_active = cancel_delayed_work(&queue->timeout.work); 1140 if (!timer_was_active || !job) 1141 queue->timeout.remaining = msecs_to_jiffies(JOB_TIMEOUT_MS); 1142 else if (time_after(qtimeout, now)) 1143 queue->timeout.remaining = qtimeout - now; 1144 else 1145 queue->timeout.remaining = 0; 1146 1147 if (WARN_ON_ONCE(queue->timeout.remaining > msecs_to_jiffies(JOB_TIMEOUT_MS))) 1148 queue->timeout.remaining = msecs_to_jiffies(JOB_TIMEOUT_MS); 1149} 1150 1151static void 1152queue_suspend_timeout(struct panthor_queue *queue) 1153{ 1154 spin_lock(&queue->fence_ctx.lock); 1155 queue_suspend_timeout_locked(queue); 1156 spin_unlock(&queue->fence_ctx.lock); 1157} 1158 1159static void 1160queue_resume_timeout(struct panthor_queue *queue) 1161{ 1162 spin_lock(&queue->fence_ctx.lock); 1163 1164 if (queue_timeout_is_suspended(queue)) { 1165 mod_delayed_work(queue->scheduler.timeout_wq, 1166 &queue->timeout.work, 1167 queue->timeout.remaining); 1168 1169 queue->timeout.remaining = MAX_SCHEDULE_TIMEOUT; 1170 } 1171 1172 spin_unlock(&queue->fence_ctx.lock); 1173} 1174 1175/** 1176 * cs_slot_prog_locked() - Program a queue slot 1177 * @ptdev: Device. 1178 * @csg_id: Group slot ID. 1179 * @cs_id: Queue slot ID. 1180 * 1181 * Program a queue slot with the queue information so things can start being 1182 * executed on this queue. 1183 * 1184 * The group slot must have a group bound to it already (group_bind_locked()). 1185 */ 1186static void 1187cs_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id) 1188{ 1189 struct panthor_queue *queue = ptdev->scheduler->csg_slots[csg_id].group->queues[cs_id]; 1190 struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); 1191 1192 lockdep_assert_held(&ptdev->scheduler->lock); 1193 1194 queue->iface.input->extract = queue->iface.output->extract; 1195 drm_WARN_ON(&ptdev->base, queue->iface.input->insert < queue->iface.input->extract); 1196 1197 cs_iface->input->ringbuf_base = panthor_kernel_bo_gpuva(queue->ringbuf); 1198 cs_iface->input->ringbuf_size = panthor_kernel_bo_size(queue->ringbuf); 1199 cs_iface->input->ringbuf_input = queue->iface.input_fw_va; 1200 cs_iface->input->ringbuf_output = queue->iface.output_fw_va; 1201 cs_iface->input->config = CS_CONFIG_PRIORITY(queue->priority) | 1202 CS_CONFIG_DOORBELL(queue->doorbell_id); 1203 cs_iface->input->ack_irq_mask = ~0; 1204 panthor_fw_update_reqs(cs_iface, req, 1205 CS_IDLE_SYNC_WAIT | 1206 CS_IDLE_EMPTY | 1207 CS_STATE_START | 1208 CS_EXTRACT_EVENT, 1209 CS_IDLE_SYNC_WAIT | 1210 CS_IDLE_EMPTY | 1211 CS_STATE_MASK | 1212 CS_EXTRACT_EVENT); 1213 if (queue->iface.input->insert != queue->iface.input->extract) 1214 queue_resume_timeout(queue); 1215} 1216 1217/** 1218 * cs_slot_reset_locked() - Reset a queue slot 1219 * @ptdev: Device. 1220 * @csg_id: Group slot. 1221 * @cs_id: Queue slot. 1222 * 1223 * Change the queue slot state to STOP and suspend the queue timeout if 1224 * the queue is not blocked. 1225 * 1226 * The group slot must have a group bound to it (group_bind_locked()). 1227 */ 1228static int 1229cs_slot_reset_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id) 1230{ 1231 struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); 1232 struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group; 1233 struct panthor_queue *queue = group->queues[cs_id]; 1234 1235 lockdep_assert_held(&ptdev->scheduler->lock); 1236 1237 panthor_fw_update_reqs(cs_iface, req, 1238 CS_STATE_STOP, 1239 CS_STATE_MASK); 1240 1241 queue_suspend_timeout(queue); 1242 1243 return 0; 1244} 1245 1246/** 1247 * csg_slot_sync_priority_locked() - Synchronize the group slot priority 1248 * @ptdev: Device. 1249 * @csg_id: Group slot ID. 1250 * 1251 * Group slot priority update happens asynchronously. When we receive a 1252 * %CSG_ENDPOINT_CONFIG, we know the update is effective, and can 1253 * reflect it to our panthor_csg_slot object. 1254 */ 1255static void 1256csg_slot_sync_priority_locked(struct panthor_device *ptdev, u32 csg_id) 1257{ 1258 struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id]; 1259 struct panthor_fw_csg_iface *csg_iface; 1260 u64 endpoint_req; 1261 1262 lockdep_assert_held(&ptdev->scheduler->lock); 1263 1264 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); 1265 endpoint_req = panthor_fw_csg_endpoint_req_get(ptdev, csg_iface); 1266 csg_slot->priority = CSG_EP_REQ_PRIORITY_GET(endpoint_req); 1267} 1268 1269/** 1270 * cs_slot_sync_queue_state_locked() - Synchronize the queue slot priority 1271 * @ptdev: Device. 1272 * @csg_id: Group slot. 1273 * @cs_id: Queue slot. 1274 * 1275 * Queue state is updated on group suspend or STATUS_UPDATE event. 1276 */ 1277static void 1278cs_slot_sync_queue_state_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id) 1279{ 1280 struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group; 1281 struct panthor_queue *queue = group->queues[cs_id]; 1282 struct panthor_fw_cs_iface *cs_iface = 1283 panthor_fw_get_cs_iface(group->ptdev, csg_id, cs_id); 1284 1285 u32 status_wait_cond; 1286 1287 switch (cs_iface->output->status_blocked_reason) { 1288 case CS_STATUS_BLOCKED_REASON_UNBLOCKED: 1289 if (queue->iface.input->insert == queue->iface.output->extract && 1290 cs_iface->output->status_scoreboards == 0) 1291 group->idle_queues |= BIT(cs_id); 1292 break; 1293 1294 case CS_STATUS_BLOCKED_REASON_SYNC_WAIT: 1295 if (list_empty(&group->wait_node)) { 1296 list_move_tail(&group->wait_node, 1297 &group->ptdev->scheduler->groups.waiting); 1298 } 1299 1300 /* The queue is only blocked if there's no deferred operation 1301 * pending, which can be checked through the scoreboard status. 1302 */ 1303 if (!cs_iface->output->status_scoreboards) 1304 group->blocked_queues |= BIT(cs_id); 1305 1306 queue->syncwait.gpu_va = cs_iface->output->status_wait_sync_ptr; 1307 queue->syncwait.ref = cs_iface->output->status_wait_sync_value; 1308 status_wait_cond = cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_COND_MASK; 1309 queue->syncwait.gt = status_wait_cond == CS_STATUS_WAIT_SYNC_COND_GT; 1310 if (cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_64B) { 1311 u64 sync_val_hi = cs_iface->output->status_wait_sync_value_hi; 1312 1313 queue->syncwait.sync64 = true; 1314 queue->syncwait.ref |= sync_val_hi << 32; 1315 } else { 1316 queue->syncwait.sync64 = false; 1317 } 1318 break; 1319 1320 default: 1321 /* Other reasons are not blocking. Consider the queue as runnable 1322 * in those cases. 1323 */ 1324 break; 1325 } 1326} 1327 1328static void 1329csg_slot_sync_queues_state_locked(struct panthor_device *ptdev, u32 csg_id) 1330{ 1331 struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id]; 1332 struct panthor_group *group = csg_slot->group; 1333 u32 i; 1334 1335 lockdep_assert_held(&ptdev->scheduler->lock); 1336 1337 group->idle_queues = 0; 1338 group->blocked_queues = 0; 1339 1340 for (i = 0; i < group->queue_count; i++) { 1341 if (group->queues[i]) 1342 cs_slot_sync_queue_state_locked(ptdev, csg_id, i); 1343 } 1344} 1345 1346static void 1347csg_slot_sync_state_locked(struct panthor_device *ptdev, u32 csg_id) 1348{ 1349 struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id]; 1350 struct panthor_fw_csg_iface *csg_iface; 1351 struct panthor_group *group; 1352 enum panthor_group_state new_state, old_state; 1353 u32 csg_state; 1354 1355 lockdep_assert_held(&ptdev->scheduler->lock); 1356 1357 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); 1358 group = csg_slot->group; 1359 1360 if (!group) 1361 return; 1362 1363 old_state = group->state; 1364 csg_state = csg_iface->output->ack & CSG_STATE_MASK; 1365 switch (csg_state) { 1366 case CSG_STATE_START: 1367 case CSG_STATE_RESUME: 1368 new_state = PANTHOR_CS_GROUP_ACTIVE; 1369 break; 1370 case CSG_STATE_TERMINATE: 1371 new_state = PANTHOR_CS_GROUP_TERMINATED; 1372 break; 1373 case CSG_STATE_SUSPEND: 1374 new_state = PANTHOR_CS_GROUP_SUSPENDED; 1375 break; 1376 default: 1377 /* The unknown state might be caused by a FW state corruption, 1378 * which means the group metadata can't be trusted anymore, and 1379 * the SUSPEND operation might propagate the corruption to the 1380 * suspend buffers. Flag the group state as unknown to make 1381 * sure it's unusable after that point. 1382 */ 1383 drm_err(&ptdev->base, "Invalid state on CSG %d (state=%d)", 1384 csg_id, csg_state); 1385 new_state = PANTHOR_CS_GROUP_UNKNOWN_STATE; 1386 break; 1387 } 1388 1389 if (old_state == new_state) 1390 return; 1391 1392 /* The unknown state might be caused by a FW issue, reset the FW to 1393 * take a fresh start. 1394 */ 1395 if (new_state == PANTHOR_CS_GROUP_UNKNOWN_STATE) 1396 panthor_device_schedule_reset(ptdev); 1397 1398 if (new_state == PANTHOR_CS_GROUP_SUSPENDED) 1399 csg_slot_sync_queues_state_locked(ptdev, csg_id); 1400 1401 if (old_state == PANTHOR_CS_GROUP_ACTIVE) { 1402 u32 i; 1403 1404 /* Reset the queue slots so we start from a clean 1405 * state when starting/resuming a new group on this 1406 * CSG slot. No wait needed here, and no ringbell 1407 * either, since the CS slot will only be re-used 1408 * on the next CSG start operation. 1409 */ 1410 for (i = 0; i < group->queue_count; i++) { 1411 if (group->queues[i]) 1412 cs_slot_reset_locked(ptdev, csg_id, i); 1413 } 1414 } 1415 1416 group->state = new_state; 1417} 1418 1419static int 1420csg_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 priority) 1421{ 1422 struct panthor_fw_csg_iface *csg_iface; 1423 struct panthor_csg_slot *csg_slot; 1424 struct panthor_group *group; 1425 u32 queue_mask = 0, i; 1426 u64 endpoint_req; 1427 1428 lockdep_assert_held(&ptdev->scheduler->lock); 1429 1430 if (priority > MAX_CSG_PRIO) 1431 return -EINVAL; 1432 1433 if (drm_WARN_ON(&ptdev->base, csg_id >= MAX_CSGS)) 1434 return -EINVAL; 1435 1436 csg_slot = &ptdev->scheduler->csg_slots[csg_id]; 1437 group = csg_slot->group; 1438 if (!group || group->state == PANTHOR_CS_GROUP_ACTIVE) 1439 return 0; 1440 1441 csg_iface = panthor_fw_get_csg_iface(group->ptdev, csg_id); 1442 1443 for (i = 0; i < group->queue_count; i++) { 1444 if (group->queues[i]) { 1445 cs_slot_prog_locked(ptdev, csg_id, i); 1446 queue_mask |= BIT(i); 1447 } 1448 } 1449 1450 csg_iface->input->allow_compute = group->compute_core_mask; 1451 csg_iface->input->allow_fragment = group->fragment_core_mask; 1452 csg_iface->input->allow_other = group->tiler_core_mask; 1453 endpoint_req = CSG_EP_REQ_COMPUTE(group->max_compute_cores) | 1454 CSG_EP_REQ_FRAGMENT(group->max_fragment_cores) | 1455 CSG_EP_REQ_TILER(group->max_tiler_cores) | 1456 CSG_EP_REQ_PRIORITY(priority); 1457 panthor_fw_csg_endpoint_req_set(ptdev, csg_iface, endpoint_req); 1458 1459 csg_iface->input->config = panthor_vm_as(group->vm); 1460 1461 if (group->suspend_buf) 1462 csg_iface->input->suspend_buf = panthor_kernel_bo_gpuva(group->suspend_buf); 1463 else 1464 csg_iface->input->suspend_buf = 0; 1465 1466 if (group->protm_suspend_buf) { 1467 csg_iface->input->protm_suspend_buf = 1468 panthor_kernel_bo_gpuva(group->protm_suspend_buf); 1469 } else { 1470 csg_iface->input->protm_suspend_buf = 0; 1471 } 1472 1473 csg_iface->input->ack_irq_mask = ~0; 1474 panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, queue_mask); 1475 return 0; 1476} 1477 1478static void 1479cs_slot_process_fatal_event_locked(struct panthor_device *ptdev, 1480 u32 csg_id, u32 cs_id) 1481{ 1482 struct panthor_scheduler *sched = ptdev->scheduler; 1483 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; 1484 struct panthor_group *group = csg_slot->group; 1485 struct panthor_fw_cs_iface *cs_iface; 1486 u32 fatal; 1487 u64 info; 1488 1489 lockdep_assert_held(&sched->lock); 1490 1491 cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); 1492 fatal = cs_iface->output->fatal; 1493 info = cs_iface->output->fatal_info; 1494 1495 if (group) { 1496 drm_warn(&ptdev->base, "CS_FATAL: pid=%d, comm=%s\n", 1497 group->task_info.pid, group->task_info.comm); 1498 1499 group->fatal_queues |= BIT(cs_id); 1500 } 1501 1502 if (CS_EXCEPTION_TYPE(fatal) == DRM_PANTHOR_EXCEPTION_CS_UNRECOVERABLE) { 1503 /* If this exception is unrecoverable, queue a reset, and make 1504 * sure we stop scheduling groups until the reset has happened. 1505 */ 1506 panthor_device_schedule_reset(ptdev); 1507 cancel_delayed_work(&sched->tick_work); 1508 } else { 1509 sched_queue_delayed_work(sched, tick, 0); 1510 } 1511 1512 drm_warn(&ptdev->base, 1513 "CSG slot %d CS slot: %d\n" 1514 "CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n" 1515 "CS_FATAL.EXCEPTION_DATA: 0x%x\n" 1516 "CS_FATAL_INFO.EXCEPTION_DATA: 0x%llx\n", 1517 csg_id, cs_id, 1518 (unsigned int)CS_EXCEPTION_TYPE(fatal), 1519 panthor_exception_name(ptdev, CS_EXCEPTION_TYPE(fatal)), 1520 (unsigned int)CS_EXCEPTION_DATA(fatal), 1521 info); 1522} 1523 1524static void 1525cs_slot_process_fault_event_locked(struct panthor_device *ptdev, 1526 u32 csg_id, u32 cs_id) 1527{ 1528 struct panthor_scheduler *sched = ptdev->scheduler; 1529 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; 1530 struct panthor_group *group = csg_slot->group; 1531 struct panthor_queue *queue = group && cs_id < group->queue_count ? 1532 group->queues[cs_id] : NULL; 1533 struct panthor_fw_cs_iface *cs_iface; 1534 u32 fault; 1535 u64 info; 1536 1537 lockdep_assert_held(&sched->lock); 1538 1539 cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); 1540 fault = cs_iface->output->fault; 1541 info = cs_iface->output->fault_info; 1542 1543 if (queue) { 1544 u64 cs_extract = queue->iface.output->extract; 1545 struct panthor_job *job; 1546 1547 spin_lock(&queue->fence_ctx.lock); 1548 list_for_each_entry(job, &queue->fence_ctx.in_flight_jobs, node) { 1549 if (cs_extract >= job->ringbuf.end) 1550 continue; 1551 1552 if (cs_extract < job->ringbuf.start) 1553 break; 1554 1555 dma_fence_set_error(job->done_fence, -EINVAL); 1556 } 1557 spin_unlock(&queue->fence_ctx.lock); 1558 } 1559 1560 if (group) { 1561 drm_warn(&ptdev->base, "CS_FAULT: pid=%d, comm=%s\n", 1562 group->task_info.pid, group->task_info.comm); 1563 } 1564 1565 drm_warn(&ptdev->base, 1566 "CSG slot %d CS slot: %d\n" 1567 "CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n" 1568 "CS_FAULT.EXCEPTION_DATA: 0x%x\n" 1569 "CS_FAULT_INFO.EXCEPTION_DATA: 0x%llx\n", 1570 csg_id, cs_id, 1571 (unsigned int)CS_EXCEPTION_TYPE(fault), 1572 panthor_exception_name(ptdev, CS_EXCEPTION_TYPE(fault)), 1573 (unsigned int)CS_EXCEPTION_DATA(fault), 1574 info); 1575} 1576 1577static int group_process_tiler_oom(struct panthor_group *group, u32 cs_id) 1578{ 1579 struct panthor_device *ptdev = group->ptdev; 1580 struct panthor_scheduler *sched = ptdev->scheduler; 1581 u32 renderpasses_in_flight, pending_frag_count; 1582 struct panthor_heap_pool *heaps = NULL; 1583 u64 heap_address, new_chunk_va = 0; 1584 u32 vt_start, vt_end, frag_end; 1585 int ret, csg_id; 1586 1587 mutex_lock(&sched->lock); 1588 csg_id = group->csg_id; 1589 if (csg_id >= 0) { 1590 struct panthor_fw_cs_iface *cs_iface; 1591 1592 cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); 1593 heaps = panthor_vm_get_heap_pool(group->vm, false); 1594 heap_address = cs_iface->output->heap_address; 1595 vt_start = cs_iface->output->heap_vt_start; 1596 vt_end = cs_iface->output->heap_vt_end; 1597 frag_end = cs_iface->output->heap_frag_end; 1598 renderpasses_in_flight = vt_start - frag_end; 1599 pending_frag_count = vt_end - frag_end; 1600 } 1601 mutex_unlock(&sched->lock); 1602 1603 /* The group got scheduled out, we stop here. We will get a new tiler OOM event 1604 * when it's scheduled again. 1605 */ 1606 if (unlikely(csg_id < 0)) 1607 return 0; 1608 1609 if (IS_ERR(heaps) || frag_end > vt_end || vt_end >= vt_start) { 1610 ret = -EINVAL; 1611 } else { 1612 /* We do the allocation without holding the scheduler lock to avoid 1613 * blocking the scheduling. 1614 */ 1615 ret = panthor_heap_grow(heaps, heap_address, 1616 renderpasses_in_flight, 1617 pending_frag_count, &new_chunk_va); 1618 } 1619 1620 /* If the heap context doesn't have memory for us, we want to let the 1621 * FW try to reclaim memory by waiting for fragment jobs to land or by 1622 * executing the tiler OOM exception handler, which is supposed to 1623 * implement incremental rendering. 1624 */ 1625 if (ret && ret != -ENOMEM) { 1626 drm_warn(&ptdev->base, "Failed to extend the tiler heap\n"); 1627 group->fatal_queues |= BIT(cs_id); 1628 sched_queue_delayed_work(sched, tick, 0); 1629 goto out_put_heap_pool; 1630 } 1631 1632 mutex_lock(&sched->lock); 1633 csg_id = group->csg_id; 1634 if (csg_id >= 0) { 1635 struct panthor_fw_csg_iface *csg_iface; 1636 struct panthor_fw_cs_iface *cs_iface; 1637 1638 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); 1639 cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); 1640 1641 cs_iface->input->heap_start = new_chunk_va; 1642 cs_iface->input->heap_end = new_chunk_va; 1643 panthor_fw_update_reqs(cs_iface, req, cs_iface->output->ack, CS_TILER_OOM); 1644 panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, BIT(cs_id)); 1645 panthor_fw_ring_csg_doorbells(ptdev, BIT(csg_id)); 1646 } 1647 mutex_unlock(&sched->lock); 1648 1649 /* We allocated a chunck, but couldn't link it to the heap 1650 * context because the group was scheduled out while we were 1651 * allocating memory. We need to return this chunk to the heap. 1652 */ 1653 if (unlikely(csg_id < 0 && new_chunk_va)) 1654 panthor_heap_return_chunk(heaps, heap_address, new_chunk_va); 1655 1656 ret = 0; 1657 1658out_put_heap_pool: 1659 panthor_heap_pool_put(heaps); 1660 return ret; 1661} 1662 1663static void group_tiler_oom_work(struct work_struct *work) 1664{ 1665 struct panthor_group *group = 1666 container_of(work, struct panthor_group, tiler_oom_work); 1667 u32 tiler_oom = atomic_xchg(&group->tiler_oom, 0); 1668 1669 while (tiler_oom) { 1670 u32 cs_id = ffs(tiler_oom) - 1; 1671 1672 group_process_tiler_oom(group, cs_id); 1673 tiler_oom &= ~BIT(cs_id); 1674 } 1675 1676 group_put(group); 1677} 1678 1679static void 1680cs_slot_process_tiler_oom_event_locked(struct panthor_device *ptdev, 1681 u32 csg_id, u32 cs_id) 1682{ 1683 struct panthor_scheduler *sched = ptdev->scheduler; 1684 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; 1685 struct panthor_group *group = csg_slot->group; 1686 1687 lockdep_assert_held(&sched->lock); 1688 1689 if (drm_WARN_ON(&ptdev->base, !group)) 1690 return; 1691 1692 atomic_or(BIT(cs_id), &group->tiler_oom); 1693 1694 /* We don't use group_queue_work() here because we want to queue the 1695 * work item to the heap_alloc_wq. 1696 */ 1697 group_get(group); 1698 if (!queue_work(sched->heap_alloc_wq, &group->tiler_oom_work)) 1699 group_put(group); 1700} 1701 1702static bool cs_slot_process_irq_locked(struct panthor_device *ptdev, 1703 u32 csg_id, u32 cs_id) 1704{ 1705 struct panthor_fw_cs_iface *cs_iface; 1706 u32 req, ack, events; 1707 1708 lockdep_assert_held(&ptdev->scheduler->lock); 1709 1710 cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); 1711 req = cs_iface->input->req; 1712 ack = cs_iface->output->ack; 1713 events = (req ^ ack) & CS_EVT_MASK; 1714 1715 if (events & CS_FATAL) 1716 cs_slot_process_fatal_event_locked(ptdev, csg_id, cs_id); 1717 1718 if (events & CS_FAULT) 1719 cs_slot_process_fault_event_locked(ptdev, csg_id, cs_id); 1720 1721 if (events & CS_TILER_OOM) 1722 cs_slot_process_tiler_oom_event_locked(ptdev, csg_id, cs_id); 1723 1724 /* We don't acknowledge the TILER_OOM event since its handling is 1725 * deferred to a separate work. 1726 */ 1727 panthor_fw_update_reqs(cs_iface, req, ack, CS_FATAL | CS_FAULT); 1728 1729 return (events & (CS_FAULT | CS_TILER_OOM)) != 0; 1730} 1731 1732static void csg_slot_sync_idle_state_locked(struct panthor_device *ptdev, u32 csg_id) 1733{ 1734 struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id]; 1735 struct panthor_fw_csg_iface *csg_iface; 1736 1737 lockdep_assert_held(&ptdev->scheduler->lock); 1738 1739 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); 1740 csg_slot->idle = csg_iface->output->status_state & CSG_STATUS_STATE_IS_IDLE; 1741} 1742 1743static void csg_slot_process_idle_event_locked(struct panthor_device *ptdev, u32 csg_id) 1744{ 1745 struct panthor_scheduler *sched = ptdev->scheduler; 1746 1747 lockdep_assert_held(&sched->lock); 1748 1749 sched->might_have_idle_groups = true; 1750 1751 /* Schedule a tick so we can evict idle groups and schedule non-idle 1752 * ones. This will also update runtime PM and devfreq busy/idle states, 1753 * so the device can lower its frequency or get suspended. 1754 */ 1755 sched_queue_delayed_work(sched, tick, 0); 1756} 1757 1758static void csg_slot_sync_update_locked(struct panthor_device *ptdev, 1759 u32 csg_id) 1760{ 1761 struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id]; 1762 struct panthor_group *group = csg_slot->group; 1763 1764 lockdep_assert_held(&ptdev->scheduler->lock); 1765 1766 if (group) 1767 group_queue_work(group, sync_upd); 1768 1769 sched_queue_work(ptdev->scheduler, sync_upd); 1770} 1771 1772static void 1773csg_slot_process_progress_timer_event_locked(struct panthor_device *ptdev, u32 csg_id) 1774{ 1775 struct panthor_scheduler *sched = ptdev->scheduler; 1776 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; 1777 struct panthor_group *group = csg_slot->group; 1778 1779 lockdep_assert_held(&sched->lock); 1780 1781 group = csg_slot->group; 1782 if (!drm_WARN_ON(&ptdev->base, !group)) { 1783 drm_warn(&ptdev->base, "CSG_PROGRESS_TIMER_EVENT: pid=%d, comm=%s\n", 1784 group->task_info.pid, group->task_info.comm); 1785 1786 group->timedout = true; 1787 } 1788 1789 drm_warn(&ptdev->base, "CSG slot %d progress timeout\n", csg_id); 1790 1791 sched_queue_delayed_work(sched, tick, 0); 1792} 1793 1794static void sched_process_csg_irq_locked(struct panthor_device *ptdev, u32 csg_id) 1795{ 1796 u32 req, ack, cs_irq_req, cs_irq_ack, cs_irqs, csg_events; 1797 struct panthor_fw_csg_iface *csg_iface; 1798 u32 ring_cs_db_mask = 0; 1799 1800 lockdep_assert_held(&ptdev->scheduler->lock); 1801 1802 if (drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count)) 1803 return; 1804 1805 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); 1806 req = READ_ONCE(csg_iface->input->req); 1807 ack = READ_ONCE(csg_iface->output->ack); 1808 cs_irq_req = READ_ONCE(csg_iface->output->cs_irq_req); 1809 cs_irq_ack = READ_ONCE(csg_iface->input->cs_irq_ack); 1810 csg_events = (req ^ ack) & CSG_EVT_MASK; 1811 1812 /* There may not be any pending CSG/CS interrupts to process */ 1813 if (req == ack && cs_irq_req == cs_irq_ack) 1814 return; 1815 1816 /* Immediately set IRQ_ACK bits to be same as the IRQ_REQ bits before 1817 * examining the CS_ACK & CS_REQ bits. This would ensure that Host 1818 * doesn't miss an interrupt for the CS in the race scenario where 1819 * whilst Host is servicing an interrupt for the CS, firmware sends 1820 * another interrupt for that CS. 1821 */ 1822 csg_iface->input->cs_irq_ack = cs_irq_req; 1823 1824 panthor_fw_update_reqs(csg_iface, req, ack, 1825 CSG_SYNC_UPDATE | 1826 CSG_IDLE | 1827 CSG_PROGRESS_TIMER_EVENT); 1828 1829 if (csg_events & CSG_IDLE) 1830 csg_slot_process_idle_event_locked(ptdev, csg_id); 1831 1832 if (csg_events & CSG_PROGRESS_TIMER_EVENT) 1833 csg_slot_process_progress_timer_event_locked(ptdev, csg_id); 1834 1835 cs_irqs = cs_irq_req ^ cs_irq_ack; 1836 while (cs_irqs) { 1837 u32 cs_id = ffs(cs_irqs) - 1; 1838 1839 if (cs_slot_process_irq_locked(ptdev, csg_id, cs_id)) 1840 ring_cs_db_mask |= BIT(cs_id); 1841 1842 cs_irqs &= ~BIT(cs_id); 1843 } 1844 1845 if (csg_events & CSG_SYNC_UPDATE) 1846 csg_slot_sync_update_locked(ptdev, csg_id); 1847 1848 if (ring_cs_db_mask) 1849 panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, ring_cs_db_mask); 1850 1851 panthor_fw_ring_csg_doorbells(ptdev, BIT(csg_id)); 1852} 1853 1854static void sched_process_idle_event_locked(struct panthor_device *ptdev) 1855{ 1856 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); 1857 1858 lockdep_assert_held(&ptdev->scheduler->lock); 1859 1860 /* Acknowledge the idle event and schedule a tick. */ 1861 panthor_fw_update_reqs(glb_iface, req, glb_iface->output->ack, GLB_IDLE); 1862 sched_queue_delayed_work(ptdev->scheduler, tick, 0); 1863} 1864 1865/** 1866 * sched_process_global_irq_locked() - Process the scheduling part of a global IRQ 1867 * @ptdev: Device. 1868 */ 1869static void sched_process_global_irq_locked(struct panthor_device *ptdev) 1870{ 1871 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); 1872 u32 req, ack, evts; 1873 1874 lockdep_assert_held(&ptdev->scheduler->lock); 1875 1876 req = READ_ONCE(glb_iface->input->req); 1877 ack = READ_ONCE(glb_iface->output->ack); 1878 evts = (req ^ ack) & GLB_EVT_MASK; 1879 1880 if (evts & GLB_IDLE) 1881 sched_process_idle_event_locked(ptdev); 1882} 1883 1884static void process_fw_events_work(struct work_struct *work) 1885{ 1886 struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler, 1887 fw_events_work); 1888 u32 events = atomic_xchg(&sched->fw_events, 0); 1889 struct panthor_device *ptdev = sched->ptdev; 1890 1891 mutex_lock(&sched->lock); 1892 1893 if (events & JOB_INT_GLOBAL_IF) { 1894 sched_process_global_irq_locked(ptdev); 1895 events &= ~JOB_INT_GLOBAL_IF; 1896 } 1897 1898 while (events) { 1899 u32 csg_id = ffs(events) - 1; 1900 1901 sched_process_csg_irq_locked(ptdev, csg_id); 1902 events &= ~BIT(csg_id); 1903 } 1904 1905 mutex_unlock(&sched->lock); 1906} 1907 1908/** 1909 * panthor_sched_report_fw_events() - Report FW events to the scheduler. 1910 */ 1911void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events) 1912{ 1913 if (!ptdev->scheduler) 1914 return; 1915 1916 atomic_or(events, &ptdev->scheduler->fw_events); 1917 sched_queue_work(ptdev->scheduler, fw_events); 1918} 1919 1920static const char *fence_get_driver_name(struct dma_fence *fence) 1921{ 1922 return "panthor"; 1923} 1924 1925static const char *queue_fence_get_timeline_name(struct dma_fence *fence) 1926{ 1927 return "queue-fence"; 1928} 1929 1930static const struct dma_fence_ops panthor_queue_fence_ops = { 1931 .get_driver_name = fence_get_driver_name, 1932 .get_timeline_name = queue_fence_get_timeline_name, 1933}; 1934 1935struct panthor_csg_slots_upd_ctx { 1936 u32 update_mask; 1937 u32 timedout_mask; 1938 struct { 1939 u32 value; 1940 u32 mask; 1941 } requests[MAX_CSGS]; 1942}; 1943 1944static void csgs_upd_ctx_init(struct panthor_csg_slots_upd_ctx *ctx) 1945{ 1946 memset(ctx, 0, sizeof(*ctx)); 1947} 1948 1949static void csgs_upd_ctx_queue_reqs(struct panthor_device *ptdev, 1950 struct panthor_csg_slots_upd_ctx *ctx, 1951 u32 csg_id, u32 value, u32 mask) 1952{ 1953 if (drm_WARN_ON(&ptdev->base, !mask) || 1954 drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count)) 1955 return; 1956 1957 ctx->requests[csg_id].value = (ctx->requests[csg_id].value & ~mask) | (value & mask); 1958 ctx->requests[csg_id].mask |= mask; 1959 ctx->update_mask |= BIT(csg_id); 1960} 1961 1962static int csgs_upd_ctx_apply_locked(struct panthor_device *ptdev, 1963 struct panthor_csg_slots_upd_ctx *ctx) 1964{ 1965 struct panthor_scheduler *sched = ptdev->scheduler; 1966 u32 update_slots = ctx->update_mask; 1967 1968 lockdep_assert_held(&sched->lock); 1969 1970 if (!ctx->update_mask) 1971 return 0; 1972 1973 while (update_slots) { 1974 struct panthor_fw_csg_iface *csg_iface; 1975 u32 csg_id = ffs(update_slots) - 1; 1976 1977 update_slots &= ~BIT(csg_id); 1978 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); 1979 panthor_fw_update_reqs(csg_iface, req, 1980 ctx->requests[csg_id].value, 1981 ctx->requests[csg_id].mask); 1982 } 1983 1984 panthor_fw_ring_csg_doorbells(ptdev, ctx->update_mask); 1985 1986 update_slots = ctx->update_mask; 1987 while (update_slots) { 1988 struct panthor_fw_csg_iface *csg_iface; 1989 u32 csg_id = ffs(update_slots) - 1; 1990 u32 req_mask = ctx->requests[csg_id].mask, acked; 1991 int ret; 1992 1993 update_slots &= ~BIT(csg_id); 1994 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); 1995 1996 ret = panthor_fw_csg_wait_acks(ptdev, csg_id, req_mask, &acked, 100); 1997 1998 if (acked & CSG_ENDPOINT_CONFIG) 1999 csg_slot_sync_priority_locked(ptdev, csg_id); 2000 2001 if (acked & CSG_STATE_MASK) 2002 csg_slot_sync_state_locked(ptdev, csg_id); 2003 2004 if (acked & CSG_STATUS_UPDATE) { 2005 csg_slot_sync_queues_state_locked(ptdev, csg_id); 2006 csg_slot_sync_idle_state_locked(ptdev, csg_id); 2007 } 2008 2009 if (ret && acked != req_mask && 2010 ((csg_iface->input->req ^ csg_iface->output->ack) & req_mask) != 0) { 2011 drm_err(&ptdev->base, "CSG %d update request timedout", csg_id); 2012 ctx->timedout_mask |= BIT(csg_id); 2013 } 2014 } 2015 2016 if (ctx->timedout_mask) 2017 return -ETIMEDOUT; 2018 2019 return 0; 2020} 2021 2022struct panthor_sched_tick_ctx { 2023 struct list_head old_groups[PANTHOR_CSG_PRIORITY_COUNT]; 2024 struct list_head groups[PANTHOR_CSG_PRIORITY_COUNT]; 2025 u32 idle_group_count; 2026 u32 group_count; 2027 struct panthor_vm *vms[MAX_CS_PER_CSG]; 2028 u32 as_count; 2029 bool immediate_tick; 2030 bool stop_tick; 2031 u32 csg_upd_failed_mask; 2032}; 2033 2034static bool 2035tick_ctx_is_full(const struct panthor_scheduler *sched, 2036 const struct panthor_sched_tick_ctx *ctx) 2037{ 2038 return ctx->group_count == sched->csg_slot_count; 2039} 2040 2041static void 2042tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched, 2043 struct panthor_sched_tick_ctx *ctx, 2044 struct list_head *queue, 2045 bool skip_idle_groups, 2046 bool owned_by_tick_ctx) 2047{ 2048 struct panthor_group *group, *tmp; 2049 2050 if (tick_ctx_is_full(sched, ctx)) 2051 return; 2052 2053 list_for_each_entry_safe(group, tmp, queue, run_node) { 2054 u32 i; 2055 2056 if (!group_can_run(group)) 2057 continue; 2058 2059 if (skip_idle_groups && group_is_idle(group)) 2060 continue; 2061 2062 for (i = 0; i < ctx->as_count; i++) { 2063 if (ctx->vms[i] == group->vm) 2064 break; 2065 } 2066 2067 if (i == ctx->as_count && ctx->as_count == sched->as_slot_count) 2068 continue; 2069 2070 if (!owned_by_tick_ctx) 2071 group_get(group); 2072 2073 ctx->group_count++; 2074 2075 /* If we have more than one active group with the same priority, 2076 * we need to keep ticking to rotate the CSG priority. 2077 */ 2078 if (group_is_idle(group)) 2079 ctx->idle_group_count++; 2080 else if (!list_empty(&ctx->groups[group->priority])) 2081 ctx->stop_tick = false; 2082 2083 list_move_tail(&group->run_node, &ctx->groups[group->priority]); 2084 2085 if (i == ctx->as_count) 2086 ctx->vms[ctx->as_count++] = group->vm; 2087 2088 if (tick_ctx_is_full(sched, ctx)) 2089 return; 2090 } 2091} 2092 2093static void 2094tick_ctx_insert_old_group(struct panthor_scheduler *sched, 2095 struct panthor_sched_tick_ctx *ctx, 2096 struct panthor_group *group) 2097{ 2098 struct panthor_csg_slot *csg_slot = &sched->csg_slots[group->csg_id]; 2099 struct panthor_group *other_group; 2100 2101 /* Class groups in descending priority order so we can easily rotate. */ 2102 list_for_each_entry(other_group, 2103 &ctx->old_groups[csg_slot->group->priority], 2104 run_node) { 2105 struct panthor_csg_slot *other_csg_slot = &sched->csg_slots[other_group->csg_id]; 2106 2107 /* Our group has a higher prio than the one we're testing against, 2108 * place it just before. 2109 */ 2110 if (csg_slot->priority > other_csg_slot->priority) { 2111 list_add_tail(&group->run_node, &other_group->run_node); 2112 return; 2113 } 2114 } 2115 2116 list_add_tail(&group->run_node, &ctx->old_groups[group->priority]); 2117} 2118 2119static void 2120tick_ctx_init(struct panthor_scheduler *sched, 2121 struct panthor_sched_tick_ctx *ctx) 2122{ 2123 struct panthor_device *ptdev = sched->ptdev; 2124 struct panthor_csg_slots_upd_ctx upd_ctx; 2125 int ret; 2126 u32 i; 2127 2128 memset(ctx, 0, sizeof(*ctx)); 2129 csgs_upd_ctx_init(&upd_ctx); 2130 2131 ctx->stop_tick = true; 2132 for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) { 2133 INIT_LIST_HEAD(&ctx->groups[i]); 2134 INIT_LIST_HEAD(&ctx->old_groups[i]); 2135 } 2136 2137 for (i = 0; i < sched->csg_slot_count; i++) { 2138 struct panthor_csg_slot *csg_slot = &sched->csg_slots[i]; 2139 struct panthor_group *group = csg_slot->group; 2140 struct panthor_fw_csg_iface *csg_iface; 2141 2142 if (!group) 2143 continue; 2144 2145 csg_iface = panthor_fw_get_csg_iface(ptdev, i); 2146 group_get(group); 2147 2148 /* If there was unhandled faults on the VM, force processing of 2149 * CSG IRQs, so we can flag the faulty queue. 2150 */ 2151 if (panthor_vm_has_unhandled_faults(group->vm)) { 2152 sched_process_csg_irq_locked(ptdev, i); 2153 2154 /* No fatal fault reported, flag all queues as faulty. */ 2155 if (!group->fatal_queues) 2156 group->fatal_queues |= GENMASK(group->queue_count - 1, 0); 2157 } 2158 2159 tick_ctx_insert_old_group(sched, ctx, group); 2160 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i, 2161 csg_iface->output->ack ^ CSG_STATUS_UPDATE, 2162 CSG_STATUS_UPDATE); 2163 } 2164 2165 ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx); 2166 if (ret) { 2167 panthor_device_schedule_reset(ptdev); 2168 ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask; 2169 } 2170} 2171 2172static void 2173group_term_post_processing(struct panthor_group *group) 2174{ 2175 struct panthor_job *job, *tmp; 2176 LIST_HEAD(faulty_jobs); 2177 bool cookie; 2178 u32 i = 0; 2179 2180 if (drm_WARN_ON(&group->ptdev->base, group_can_run(group))) 2181 return; 2182 2183 cookie = dma_fence_begin_signalling(); 2184 for (i = 0; i < group->queue_count; i++) { 2185 struct panthor_queue *queue = group->queues[i]; 2186 struct panthor_syncobj_64b *syncobj; 2187 int err; 2188 2189 if (group->fatal_queues & BIT(i)) 2190 err = -EINVAL; 2191 else if (group->timedout) 2192 err = -ETIMEDOUT; 2193 else 2194 err = -ECANCELED; 2195 2196 if (!queue) 2197 continue; 2198 2199 spin_lock(&queue->fence_ctx.lock); 2200 list_for_each_entry_safe(job, tmp, &queue->fence_ctx.in_flight_jobs, node) { 2201 list_move_tail(&job->node, &faulty_jobs); 2202 dma_fence_set_error(job->done_fence, err); 2203 dma_fence_signal_locked(job->done_fence); 2204 } 2205 spin_unlock(&queue->fence_ctx.lock); 2206 2207 /* Manually update the syncobj seqno to unblock waiters. */ 2208 syncobj = group->syncobjs->kmap + (i * sizeof(*syncobj)); 2209 syncobj->status = ~0; 2210 syncobj->seqno = atomic64_read(&queue->fence_ctx.seqno); 2211 sched_queue_work(group->ptdev->scheduler, sync_upd); 2212 } 2213 dma_fence_end_signalling(cookie); 2214 2215 list_for_each_entry_safe(job, tmp, &faulty_jobs, node) { 2216 list_del_init(&job->node); 2217 panthor_job_put(&job->base); 2218 } 2219} 2220 2221static void group_term_work(struct work_struct *work) 2222{ 2223 struct panthor_group *group = 2224 container_of(work, struct panthor_group, term_work); 2225 2226 group_term_post_processing(group); 2227 group_put(group); 2228} 2229 2230static void 2231tick_ctx_cleanup(struct panthor_scheduler *sched, 2232 struct panthor_sched_tick_ctx *ctx) 2233{ 2234 struct panthor_device *ptdev = sched->ptdev; 2235 struct panthor_group *group, *tmp; 2236 u32 i; 2237 2238 for (i = 0; i < ARRAY_SIZE(ctx->old_groups); i++) { 2239 list_for_each_entry_safe(group, tmp, &ctx->old_groups[i], run_node) { 2240 /* If everything went fine, we should only have groups 2241 * to be terminated in the old_groups lists. 2242 */ 2243 drm_WARN_ON(&ptdev->base, !ctx->csg_upd_failed_mask && 2244 group_can_run(group)); 2245 2246 if (!group_can_run(group)) { 2247 list_del_init(&group->run_node); 2248 list_del_init(&group->wait_node); 2249 group_queue_work(group, term); 2250 } else if (group->csg_id >= 0) { 2251 list_del_init(&group->run_node); 2252 } else { 2253 list_move(&group->run_node, 2254 group_is_idle(group) ? 2255 &sched->groups.idle[group->priority] : 2256 &sched->groups.runnable[group->priority]); 2257 } 2258 group_put(group); 2259 } 2260 } 2261 2262 for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) { 2263 /* If everything went fine, the groups to schedule lists should 2264 * be empty. 2265 */ 2266 drm_WARN_ON(&ptdev->base, 2267 !ctx->csg_upd_failed_mask && !list_empty(&ctx->groups[i])); 2268 2269 list_for_each_entry_safe(group, tmp, &ctx->groups[i], run_node) { 2270 if (group->csg_id >= 0) { 2271 list_del_init(&group->run_node); 2272 } else { 2273 list_move(&group->run_node, 2274 group_is_idle(group) ? 2275 &sched->groups.idle[group->priority] : 2276 &sched->groups.runnable[group->priority]); 2277 } 2278 group_put(group); 2279 } 2280 } 2281} 2282 2283static void 2284tick_ctx_apply(struct panthor_scheduler *sched, struct panthor_sched_tick_ctx *ctx) 2285{ 2286 struct panthor_group *group, *tmp; 2287 struct panthor_device *ptdev = sched->ptdev; 2288 struct panthor_csg_slot *csg_slot; 2289 int prio, new_csg_prio = MAX_CSG_PRIO, i; 2290 u32 free_csg_slots = 0; 2291 struct panthor_csg_slots_upd_ctx upd_ctx; 2292 int ret; 2293 2294 csgs_upd_ctx_init(&upd_ctx); 2295 2296 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { 2297 /* Suspend or terminate evicted groups. */ 2298 list_for_each_entry(group, &ctx->old_groups[prio], run_node) { 2299 bool term = !group_can_run(group); 2300 int csg_id = group->csg_id; 2301 2302 if (drm_WARN_ON(&ptdev->base, csg_id < 0)) 2303 continue; 2304 2305 csg_slot = &sched->csg_slots[csg_id]; 2306 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, 2307 term ? CSG_STATE_TERMINATE : CSG_STATE_SUSPEND, 2308 CSG_STATE_MASK); 2309 } 2310 2311 /* Update priorities on already running groups. */ 2312 list_for_each_entry(group, &ctx->groups[prio], run_node) { 2313 struct panthor_fw_csg_iface *csg_iface; 2314 int csg_id = group->csg_id; 2315 2316 if (csg_id < 0) { 2317 new_csg_prio--; 2318 continue; 2319 } 2320 2321 csg_slot = &sched->csg_slots[csg_id]; 2322 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); 2323 if (csg_slot->priority == new_csg_prio) { 2324 new_csg_prio--; 2325 continue; 2326 } 2327 2328 panthor_fw_csg_endpoint_req_update(ptdev, csg_iface, 2329 CSG_EP_REQ_PRIORITY(new_csg_prio), 2330 CSG_EP_REQ_PRIORITY_MASK); 2331 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, 2332 csg_iface->output->ack ^ CSG_ENDPOINT_CONFIG, 2333 CSG_ENDPOINT_CONFIG); 2334 new_csg_prio--; 2335 } 2336 } 2337 2338 ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx); 2339 if (ret) { 2340 panthor_device_schedule_reset(ptdev); 2341 ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask; 2342 return; 2343 } 2344 2345 /* Unbind evicted groups. */ 2346 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { 2347 list_for_each_entry(group, &ctx->old_groups[prio], run_node) { 2348 /* This group is gone. Process interrupts to clear 2349 * any pending interrupts before we start the new 2350 * group. 2351 */ 2352 if (group->csg_id >= 0) 2353 sched_process_csg_irq_locked(ptdev, group->csg_id); 2354 2355 group_unbind_locked(group); 2356 } 2357 } 2358 2359 for (i = 0; i < sched->csg_slot_count; i++) { 2360 if (!sched->csg_slots[i].group) 2361 free_csg_slots |= BIT(i); 2362 } 2363 2364 csgs_upd_ctx_init(&upd_ctx); 2365 new_csg_prio = MAX_CSG_PRIO; 2366 2367 /* Start new groups. */ 2368 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { 2369 list_for_each_entry(group, &ctx->groups[prio], run_node) { 2370 int csg_id = group->csg_id; 2371 struct panthor_fw_csg_iface *csg_iface; 2372 2373 if (csg_id >= 0) { 2374 new_csg_prio--; 2375 continue; 2376 } 2377 2378 csg_id = ffs(free_csg_slots) - 1; 2379 if (drm_WARN_ON(&ptdev->base, csg_id < 0)) 2380 break; 2381 2382 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); 2383 csg_slot = &sched->csg_slots[csg_id]; 2384 group_bind_locked(group, csg_id); 2385 csg_slot_prog_locked(ptdev, csg_id, new_csg_prio--); 2386 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, 2387 group->state == PANTHOR_CS_GROUP_SUSPENDED ? 2388 CSG_STATE_RESUME : CSG_STATE_START, 2389 CSG_STATE_MASK); 2390 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, 2391 csg_iface->output->ack ^ CSG_ENDPOINT_CONFIG, 2392 CSG_ENDPOINT_CONFIG); 2393 free_csg_slots &= ~BIT(csg_id); 2394 } 2395 } 2396 2397 ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx); 2398 if (ret) { 2399 panthor_device_schedule_reset(ptdev); 2400 ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask; 2401 return; 2402 } 2403 2404 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { 2405 list_for_each_entry_safe(group, tmp, &ctx->groups[prio], run_node) { 2406 list_del_init(&group->run_node); 2407 2408 /* If the group has been destroyed while we were 2409 * scheduling, ask for an immediate tick to 2410 * re-evaluate as soon as possible and get rid of 2411 * this dangling group. 2412 */ 2413 if (group->destroyed) 2414 ctx->immediate_tick = true; 2415 group_put(group); 2416 } 2417 2418 /* Return evicted groups to the idle or run queues. Groups 2419 * that can no longer be run (because they've been destroyed 2420 * or experienced an unrecoverable error) will be scheduled 2421 * for destruction in tick_ctx_cleanup(). 2422 */ 2423 list_for_each_entry_safe(group, tmp, &ctx->old_groups[prio], run_node) { 2424 if (!group_can_run(group)) 2425 continue; 2426 2427 if (group_is_idle(group)) 2428 list_move_tail(&group->run_node, &sched->groups.idle[prio]); 2429 else 2430 list_move_tail(&group->run_node, &sched->groups.runnable[prio]); 2431 group_put(group); 2432 } 2433 } 2434 2435 sched->used_csg_slot_count = ctx->group_count; 2436 sched->might_have_idle_groups = ctx->idle_group_count > 0; 2437} 2438 2439static u64 2440tick_ctx_update_resched_target(struct panthor_scheduler *sched, 2441 const struct panthor_sched_tick_ctx *ctx) 2442{ 2443 u64 resched_target; 2444 2445 if (ctx->stop_tick) 2446 goto no_tick; 2447 2448 resched_target = sched->last_tick + sched->tick_period; 2449 2450 if (time_before64(sched->resched_target, sched->last_tick) || 2451 time_before64(resched_target, sched->resched_target)) 2452 sched->resched_target = resched_target; 2453 2454 return sched->resched_target - sched->last_tick; 2455 2456no_tick: 2457 sched->resched_target = U64_MAX; 2458 return U64_MAX; 2459} 2460 2461static void tick_work(struct work_struct *work) 2462{ 2463 struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler, 2464 tick_work.work); 2465 struct panthor_device *ptdev = sched->ptdev; 2466 struct panthor_sched_tick_ctx ctx; 2467 u64 resched_target = sched->resched_target; 2468 u64 remaining_jiffies = 0, resched_delay; 2469 u64 now = get_jiffies_64(); 2470 int prio, ret, cookie; 2471 bool full_tick; 2472 2473 if (!drm_dev_enter(&ptdev->base, &cookie)) 2474 return; 2475 2476 ret = panthor_device_resume_and_get(ptdev); 2477 if (drm_WARN_ON(&ptdev->base, ret)) 2478 goto out_dev_exit; 2479 2480 /* If the tick is stopped, calculate when the next tick would be */ 2481 if (resched_target == U64_MAX) 2482 resched_target = sched->last_tick + sched->tick_period; 2483 2484 if (time_before64(now, resched_target)) 2485 remaining_jiffies = resched_target - now; 2486 2487 full_tick = remaining_jiffies == 0; 2488 2489 mutex_lock(&sched->lock); 2490 if (panthor_device_reset_is_pending(sched->ptdev)) 2491 goto out_unlock; 2492 2493 tick_ctx_init(sched, &ctx); 2494 if (ctx.csg_upd_failed_mask) 2495 goto out_cleanup_ctx; 2496 2497 if (!full_tick) { 2498 /* Scheduling forced in the middle of a tick. Only RT groups 2499 * can preempt non-RT ones. Currently running RT groups can't be 2500 * preempted. 2501 */ 2502 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; 2503 prio >= 0 && !tick_ctx_is_full(sched, &ctx); 2504 prio--) { 2505 tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], 2506 true, true); 2507 if (prio == PANTHOR_CSG_PRIORITY_RT) { 2508 tick_ctx_pick_groups_from_list(sched, &ctx, 2509 &sched->groups.runnable[prio], 2510 true, false); 2511 } 2512 } 2513 } 2514 2515 /* First pick non-idle groups */ 2516 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; 2517 prio >= 0 && !tick_ctx_is_full(sched, &ctx); 2518 prio--) { 2519 struct panthor_group *old_highest_prio_group = 2520 list_first_entry_or_null(&ctx.old_groups[prio], 2521 struct panthor_group, run_node); 2522 2523 /* Pull out the group with the highest prio for rotation. */ 2524 if (old_highest_prio_group) 2525 list_del(&old_highest_prio_group->run_node); 2526 2527 /* Re-insert old active groups so they get a chance to run with higher prio. */ 2528 tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true); 2529 2530 /* Fill the remaining slots with runnable groups. */ 2531 tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.runnable[prio], 2532 true, false); 2533 2534 /* Re-insert the old group with the highest prio, and give it a chance to be 2535 * scheduled again (but with a lower prio) if there's room left. 2536 */ 2537 if (old_highest_prio_group) { 2538 list_add_tail(&old_highest_prio_group->run_node, &ctx.old_groups[prio]); 2539 tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], 2540 true, true); 2541 } 2542 } 2543 2544 /* If we have free CSG slots left, pick idle groups */ 2545 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; 2546 prio >= 0 && !tick_ctx_is_full(sched, &ctx); 2547 prio--) { 2548 /* Check the old_group queue first to avoid reprogramming the slots */ 2549 tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], false, true); 2550 tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.idle[prio], 2551 false, false); 2552 } 2553 2554 tick_ctx_apply(sched, &ctx); 2555 if (ctx.csg_upd_failed_mask) 2556 goto out_cleanup_ctx; 2557 2558 if (ctx.idle_group_count == ctx.group_count) { 2559 panthor_devfreq_record_idle(sched->ptdev); 2560 if (sched->pm.has_ref) { 2561 pm_runtime_put_autosuspend(ptdev->base.dev); 2562 sched->pm.has_ref = false; 2563 } 2564 } else { 2565 panthor_devfreq_record_busy(sched->ptdev); 2566 if (!sched->pm.has_ref) { 2567 pm_runtime_get(ptdev->base.dev); 2568 sched->pm.has_ref = true; 2569 } 2570 } 2571 2572 sched->last_tick = now; 2573 resched_delay = tick_ctx_update_resched_target(sched, &ctx); 2574 if (ctx.immediate_tick) 2575 resched_delay = 0; 2576 2577 if (resched_delay != U64_MAX) 2578 sched_queue_delayed_work(sched, tick, resched_delay); 2579 2580out_cleanup_ctx: 2581 tick_ctx_cleanup(sched, &ctx); 2582 2583out_unlock: 2584 mutex_unlock(&sched->lock); 2585 pm_runtime_mark_last_busy(ptdev->base.dev); 2586 pm_runtime_put_autosuspend(ptdev->base.dev); 2587 2588out_dev_exit: 2589 drm_dev_exit(cookie); 2590} 2591 2592static int panthor_queue_eval_syncwait(struct panthor_group *group, u8 queue_idx) 2593{ 2594 struct panthor_queue *queue = group->queues[queue_idx]; 2595 union { 2596 struct panthor_syncobj_64b sync64; 2597 struct panthor_syncobj_32b sync32; 2598 } *syncobj; 2599 bool result; 2600 u64 value; 2601 2602 syncobj = panthor_queue_get_syncwait_obj(group, queue); 2603 if (!syncobj) 2604 return -EINVAL; 2605 2606 value = queue->syncwait.sync64 ? 2607 syncobj->sync64.seqno : 2608 syncobj->sync32.seqno; 2609 2610 if (queue->syncwait.gt) 2611 result = value > queue->syncwait.ref; 2612 else 2613 result = value <= queue->syncwait.ref; 2614 2615 if (result) 2616 panthor_queue_put_syncwait_obj(queue); 2617 2618 return result; 2619} 2620 2621static void sync_upd_work(struct work_struct *work) 2622{ 2623 struct panthor_scheduler *sched = container_of(work, 2624 struct panthor_scheduler, 2625 sync_upd_work); 2626 struct panthor_group *group, *tmp; 2627 bool immediate_tick = false; 2628 2629 mutex_lock(&sched->lock); 2630 list_for_each_entry_safe(group, tmp, &sched->groups.waiting, wait_node) { 2631 u32 tested_queues = group->blocked_queues; 2632 u32 unblocked_queues = 0; 2633 2634 while (tested_queues) { 2635 u32 cs_id = ffs(tested_queues) - 1; 2636 int ret; 2637 2638 ret = panthor_queue_eval_syncwait(group, cs_id); 2639 drm_WARN_ON(&group->ptdev->base, ret < 0); 2640 if (ret) 2641 unblocked_queues |= BIT(cs_id); 2642 2643 tested_queues &= ~BIT(cs_id); 2644 } 2645 2646 if (unblocked_queues) { 2647 group->blocked_queues &= ~unblocked_queues; 2648 2649 if (group->csg_id < 0) { 2650 list_move(&group->run_node, 2651 &sched->groups.runnable[group->priority]); 2652 if (group->priority == PANTHOR_CSG_PRIORITY_RT) 2653 immediate_tick = true; 2654 } 2655 } 2656 2657 if (!group->blocked_queues) 2658 list_del_init(&group->wait_node); 2659 } 2660 mutex_unlock(&sched->lock); 2661 2662 if (immediate_tick) 2663 sched_queue_delayed_work(sched, tick, 0); 2664} 2665 2666static void sched_resume_tick(struct panthor_device *ptdev) 2667{ 2668 struct panthor_scheduler *sched = ptdev->scheduler; 2669 u64 delay_jiffies, now; 2670 2671 drm_WARN_ON(&ptdev->base, sched->resched_target != U64_MAX); 2672 2673 /* Scheduler tick was off, recalculate the resched_target based on the 2674 * last tick event, and queue the scheduler work. 2675 */ 2676 now = get_jiffies_64(); 2677 sched->resched_target = sched->last_tick + sched->tick_period; 2678 if (sched->used_csg_slot_count == sched->csg_slot_count && 2679 time_before64(now, sched->resched_target)) 2680 delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX); 2681 else 2682 delay_jiffies = 0; 2683 2684 sched_queue_delayed_work(sched, tick, delay_jiffies); 2685} 2686 2687static void group_schedule_locked(struct panthor_group *group, u32 queue_mask) 2688{ 2689 struct panthor_device *ptdev = group->ptdev; 2690 struct panthor_scheduler *sched = ptdev->scheduler; 2691 struct list_head *queue = &sched->groups.runnable[group->priority]; 2692 bool was_idle; 2693 2694 if (!group_can_run(group)) 2695 return; 2696 2697 /* All updated queues are blocked, no need to wake up the scheduler. */ 2698 if ((queue_mask & group->blocked_queues) == queue_mask) 2699 return; 2700 2701 was_idle = group_is_idle(group); 2702 group->idle_queues &= ~queue_mask; 2703 2704 /* Don't mess up with the lists if we're in a middle of a reset. */ 2705 if (atomic_read(&sched->reset.in_progress)) 2706 return; 2707 2708 if (was_idle && !group_is_idle(group)) 2709 list_move_tail(&group->run_node, queue); 2710 2711 /* RT groups are preemptive. */ 2712 if (group->priority == PANTHOR_CSG_PRIORITY_RT) { 2713 sched_queue_delayed_work(sched, tick, 0); 2714 return; 2715 } 2716 2717 /* Some groups might be idle, force an immediate tick to 2718 * re-evaluate. 2719 */ 2720 if (sched->might_have_idle_groups) { 2721 sched_queue_delayed_work(sched, tick, 0); 2722 return; 2723 } 2724 2725 /* Scheduler is ticking, nothing to do. */ 2726 if (sched->resched_target != U64_MAX) { 2727 /* If there are free slots, force immediating ticking. */ 2728 if (sched->used_csg_slot_count < sched->csg_slot_count) 2729 sched_queue_delayed_work(sched, tick, 0); 2730 2731 return; 2732 } 2733 2734 /* Scheduler tick was off, recalculate the resched_target based on the 2735 * last tick event, and queue the scheduler work. 2736 */ 2737 sched_resume_tick(ptdev); 2738} 2739 2740static void queue_stop(struct panthor_queue *queue, 2741 struct panthor_job *bad_job) 2742{ 2743 disable_delayed_work_sync(&queue->timeout.work); 2744 drm_sched_stop(&queue->scheduler, bad_job ? &bad_job->base : NULL); 2745} 2746 2747static void queue_start(struct panthor_queue *queue) 2748{ 2749 struct panthor_job *job; 2750 2751 /* Re-assign the parent fences. */ 2752 list_for_each_entry(job, &queue->scheduler.pending_list, base.list) 2753 job->base.s_fence->parent = dma_fence_get(job->done_fence); 2754 2755 enable_delayed_work(&queue->timeout.work); 2756 drm_sched_start(&queue->scheduler, 0); 2757} 2758 2759static void panthor_group_stop(struct panthor_group *group) 2760{ 2761 struct panthor_scheduler *sched = group->ptdev->scheduler; 2762 2763 lockdep_assert_held(&sched->reset.lock); 2764 2765 for (u32 i = 0; i < group->queue_count; i++) 2766 queue_stop(group->queues[i], NULL); 2767 2768 group_get(group); 2769 list_move_tail(&group->run_node, &sched->reset.stopped_groups); 2770} 2771 2772static void panthor_group_start(struct panthor_group *group) 2773{ 2774 struct panthor_scheduler *sched = group->ptdev->scheduler; 2775 2776 lockdep_assert_held(&group->ptdev->scheduler->reset.lock); 2777 2778 for (u32 i = 0; i < group->queue_count; i++) 2779 queue_start(group->queues[i]); 2780 2781 if (group_can_run(group)) { 2782 list_move_tail(&group->run_node, 2783 group_is_idle(group) ? 2784 &sched->groups.idle[group->priority] : 2785 &sched->groups.runnable[group->priority]); 2786 } else { 2787 list_del_init(&group->run_node); 2788 list_del_init(&group->wait_node); 2789 group_queue_work(group, term); 2790 } 2791 2792 group_put(group); 2793} 2794 2795static void panthor_sched_immediate_tick(struct panthor_device *ptdev) 2796{ 2797 struct panthor_scheduler *sched = ptdev->scheduler; 2798 2799 sched_queue_delayed_work(sched, tick, 0); 2800} 2801 2802/** 2803 * panthor_sched_report_mmu_fault() - Report MMU faults to the scheduler. 2804 */ 2805void panthor_sched_report_mmu_fault(struct panthor_device *ptdev) 2806{ 2807 /* Force a tick to immediately kill faulty groups. */ 2808 if (ptdev->scheduler) 2809 panthor_sched_immediate_tick(ptdev); 2810} 2811 2812void panthor_sched_prepare_for_vm_destruction(struct panthor_device *ptdev) 2813{ 2814 /* FW can write out internal state, like the heap context, during CSG 2815 * suspend. It is therefore important that the scheduler has fully 2816 * evicted any pending and related groups before VM destruction can 2817 * safely continue. Failure to do so can lead to GPU page faults. 2818 * A controlled termination of a Panthor instance involves destroying 2819 * the group(s) before the VM. This means any relevant group eviction 2820 * has already been initiated by this point, and we just need to 2821 * ensure that any pending tick_work() has been completed. 2822 */ 2823 flush_work(&ptdev->scheduler->tick_work.work); 2824} 2825 2826void panthor_sched_resume(struct panthor_device *ptdev) 2827{ 2828 /* Force a tick to re-evaluate after a resume. */ 2829 panthor_sched_immediate_tick(ptdev); 2830} 2831 2832void panthor_sched_suspend(struct panthor_device *ptdev) 2833{ 2834 struct panthor_scheduler *sched = ptdev->scheduler; 2835 struct panthor_csg_slots_upd_ctx upd_ctx; 2836 u32 suspended_slots; 2837 u32 i; 2838 2839 mutex_lock(&sched->lock); 2840 csgs_upd_ctx_init(&upd_ctx); 2841 for (i = 0; i < sched->csg_slot_count; i++) { 2842 struct panthor_csg_slot *csg_slot = &sched->csg_slots[i]; 2843 2844 if (csg_slot->group) { 2845 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i, 2846 group_can_run(csg_slot->group) ? 2847 CSG_STATE_SUSPEND : CSG_STATE_TERMINATE, 2848 CSG_STATE_MASK); 2849 } 2850 } 2851 2852 suspended_slots = upd_ctx.update_mask; 2853 2854 csgs_upd_ctx_apply_locked(ptdev, &upd_ctx); 2855 suspended_slots &= ~upd_ctx.timedout_mask; 2856 2857 if (upd_ctx.timedout_mask) { 2858 u32 slot_mask = upd_ctx.timedout_mask; 2859 2860 drm_err(&ptdev->base, "CSG suspend failed, escalating to termination"); 2861 csgs_upd_ctx_init(&upd_ctx); 2862 while (slot_mask) { 2863 u32 csg_id = ffs(slot_mask) - 1; 2864 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; 2865 2866 /* If the group was still usable before that point, we consider 2867 * it innocent. 2868 */ 2869 if (group_can_run(csg_slot->group)) 2870 csg_slot->group->innocent = true; 2871 2872 /* We consider group suspension failures as fatal and flag the 2873 * group as unusable by setting timedout=true. 2874 */ 2875 csg_slot->group->timedout = true; 2876 2877 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, 2878 CSG_STATE_TERMINATE, 2879 CSG_STATE_MASK); 2880 slot_mask &= ~BIT(csg_id); 2881 } 2882 2883 csgs_upd_ctx_apply_locked(ptdev, &upd_ctx); 2884 2885 slot_mask = upd_ctx.timedout_mask; 2886 while (slot_mask) { 2887 u32 csg_id = ffs(slot_mask) - 1; 2888 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; 2889 struct panthor_group *group = csg_slot->group; 2890 2891 /* Terminate command timedout, but the soft-reset will 2892 * automatically terminate all active groups, so let's 2893 * force the state to halted here. 2894 */ 2895 if (group->state != PANTHOR_CS_GROUP_TERMINATED) { 2896 group->state = PANTHOR_CS_GROUP_TERMINATED; 2897 2898 /* Reset the queue slots manually if the termination 2899 * request failed. 2900 */ 2901 for (i = 0; i < group->queue_count; i++) { 2902 if (group->queues[i]) 2903 cs_slot_reset_locked(ptdev, csg_id, i); 2904 } 2905 } 2906 slot_mask &= ~BIT(csg_id); 2907 } 2908 } 2909 2910 /* Flush L2 and LSC caches to make sure suspend state is up-to-date. 2911 * If the flush fails, flag all queues for termination. 2912 */ 2913 if (suspended_slots) { 2914 bool flush_caches_failed = false; 2915 u32 slot_mask = suspended_slots; 2916 2917 if (panthor_gpu_flush_caches(ptdev, CACHE_CLEAN, CACHE_CLEAN, 0)) 2918 flush_caches_failed = true; 2919 2920 while (slot_mask) { 2921 u32 csg_id = ffs(slot_mask) - 1; 2922 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; 2923 2924 if (flush_caches_failed) 2925 csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED; 2926 else 2927 csg_slot_sync_update_locked(ptdev, csg_id); 2928 2929 slot_mask &= ~BIT(csg_id); 2930 } 2931 } 2932 2933 for (i = 0; i < sched->csg_slot_count; i++) { 2934 struct panthor_csg_slot *csg_slot = &sched->csg_slots[i]; 2935 struct panthor_group *group = csg_slot->group; 2936 2937 if (!group) 2938 continue; 2939 2940 group_get(group); 2941 2942 if (group->csg_id >= 0) 2943 sched_process_csg_irq_locked(ptdev, group->csg_id); 2944 2945 group_unbind_locked(group); 2946 2947 drm_WARN_ON(&group->ptdev->base, !list_empty(&group->run_node)); 2948 2949 if (group_can_run(group)) { 2950 list_add(&group->run_node, 2951 &sched->groups.idle[group->priority]); 2952 } else { 2953 /* We don't bother stopping the scheduler if the group is 2954 * faulty, the group termination work will finish the job. 2955 */ 2956 list_del_init(&group->wait_node); 2957 group_queue_work(group, term); 2958 } 2959 group_put(group); 2960 } 2961 mutex_unlock(&sched->lock); 2962} 2963 2964void panthor_sched_pre_reset(struct panthor_device *ptdev) 2965{ 2966 struct panthor_scheduler *sched = ptdev->scheduler; 2967 struct panthor_group *group, *group_tmp; 2968 u32 i; 2969 2970 mutex_lock(&sched->reset.lock); 2971 atomic_set(&sched->reset.in_progress, true); 2972 2973 /* Cancel all scheduler works. Once this is done, these works can't be 2974 * scheduled again until the reset operation is complete. 2975 */ 2976 cancel_work_sync(&sched->sync_upd_work); 2977 cancel_delayed_work_sync(&sched->tick_work); 2978 2979 panthor_sched_suspend(ptdev); 2980 2981 /* Stop all groups that might still accept jobs, so we don't get passed 2982 * new jobs while we're resetting. 2983 */ 2984 for (i = 0; i < ARRAY_SIZE(sched->groups.runnable); i++) { 2985 /* All groups should be in the idle lists. */ 2986 drm_WARN_ON(&ptdev->base, !list_empty(&sched->groups.runnable[i])); 2987 list_for_each_entry_safe(group, group_tmp, &sched->groups.runnable[i], run_node) 2988 panthor_group_stop(group); 2989 } 2990 2991 for (i = 0; i < ARRAY_SIZE(sched->groups.idle); i++) { 2992 list_for_each_entry_safe(group, group_tmp, &sched->groups.idle[i], run_node) 2993 panthor_group_stop(group); 2994 } 2995 2996 mutex_unlock(&sched->reset.lock); 2997} 2998 2999void panthor_sched_post_reset(struct panthor_device *ptdev, bool reset_failed) 3000{ 3001 struct panthor_scheduler *sched = ptdev->scheduler; 3002 struct panthor_group *group, *group_tmp; 3003 3004 mutex_lock(&sched->reset.lock); 3005 3006 list_for_each_entry_safe(group, group_tmp, &sched->reset.stopped_groups, run_node) { 3007 /* Consider all previously running group as terminated if the 3008 * reset failed. 3009 */ 3010 if (reset_failed) 3011 group->state = PANTHOR_CS_GROUP_TERMINATED; 3012 3013 panthor_group_start(group); 3014 } 3015 3016 /* We're done resetting the GPU, clear the reset.in_progress bit so we can 3017 * kick the scheduler. 3018 */ 3019 atomic_set(&sched->reset.in_progress, false); 3020 mutex_unlock(&sched->reset.lock); 3021 3022 /* No need to queue a tick and update syncs if the reset failed. */ 3023 if (!reset_failed) { 3024 sched_queue_delayed_work(sched, tick, 0); 3025 sched_queue_work(sched, sync_upd); 3026 } 3027} 3028 3029static void update_fdinfo_stats(struct panthor_job *job) 3030{ 3031 struct panthor_group *group = job->group; 3032 struct panthor_queue *queue = group->queues[job->queue_idx]; 3033 struct panthor_gpu_usage *fdinfo = &group->fdinfo.data; 3034 struct panthor_job_profiling_data *slots = queue->profiling.slots->kmap; 3035 struct panthor_job_profiling_data *data = &slots[job->profiling.slot]; 3036 3037 scoped_guard(spinlock, &group->fdinfo.lock) { 3038 if (job->profiling.mask & PANTHOR_DEVICE_PROFILING_CYCLES) 3039 fdinfo->cycles += data->cycles.after - data->cycles.before; 3040 if (job->profiling.mask & PANTHOR_DEVICE_PROFILING_TIMESTAMP) 3041 fdinfo->time += data->time.after - data->time.before; 3042 } 3043} 3044 3045void panthor_fdinfo_gather_group_samples(struct panthor_file *pfile) 3046{ 3047 struct panthor_group_pool *gpool = pfile->groups; 3048 struct panthor_group *group; 3049 unsigned long i; 3050 3051 if (IS_ERR_OR_NULL(gpool)) 3052 return; 3053 3054 xa_lock(&gpool->xa); 3055 xa_for_each_marked(&gpool->xa, i, group, GROUP_REGISTERED) { 3056 guard(spinlock)(&group->fdinfo.lock); 3057 pfile->stats.cycles += group->fdinfo.data.cycles; 3058 pfile->stats.time += group->fdinfo.data.time; 3059 group->fdinfo.data.cycles = 0; 3060 group->fdinfo.data.time = 0; 3061 } 3062 xa_unlock(&gpool->xa); 3063} 3064 3065static bool queue_check_job_completion(struct panthor_queue *queue) 3066{ 3067 struct panthor_syncobj_64b *syncobj = NULL; 3068 struct panthor_job *job, *job_tmp; 3069 bool cookie, progress = false; 3070 LIST_HEAD(done_jobs); 3071 3072 cookie = dma_fence_begin_signalling(); 3073 spin_lock(&queue->fence_ctx.lock); 3074 list_for_each_entry_safe(job, job_tmp, &queue->fence_ctx.in_flight_jobs, node) { 3075 if (!syncobj) { 3076 struct panthor_group *group = job->group; 3077 3078 syncobj = group->syncobjs->kmap + 3079 (job->queue_idx * sizeof(*syncobj)); 3080 } 3081 3082 if (syncobj->seqno < job->done_fence->seqno) 3083 break; 3084 3085 list_move_tail(&job->node, &done_jobs); 3086 dma_fence_signal_locked(job->done_fence); 3087 } 3088 3089 if (list_empty(&queue->fence_ctx.in_flight_jobs)) { 3090 /* If we have no job left, we cancel the timer, and reset remaining 3091 * time to its default so it can be restarted next time 3092 * queue_resume_timeout() is called. 3093 */ 3094 queue_suspend_timeout_locked(queue); 3095 3096 /* If there's no job pending, we consider it progress to avoid a 3097 * spurious timeout if the timeout handler and the sync update 3098 * handler raced. 3099 */ 3100 progress = true; 3101 } else if (!list_empty(&done_jobs)) { 3102 queue_reset_timeout_locked(queue); 3103 progress = true; 3104 } 3105 spin_unlock(&queue->fence_ctx.lock); 3106 dma_fence_end_signalling(cookie); 3107 3108 list_for_each_entry_safe(job, job_tmp, &done_jobs, node) { 3109 if (job->profiling.mask) 3110 update_fdinfo_stats(job); 3111 list_del_init(&job->node); 3112 panthor_job_put(&job->base); 3113 } 3114 3115 return progress; 3116} 3117 3118static void group_sync_upd_work(struct work_struct *work) 3119{ 3120 struct panthor_group *group = 3121 container_of(work, struct panthor_group, sync_upd_work); 3122 u32 queue_idx; 3123 bool cookie; 3124 3125 cookie = dma_fence_begin_signalling(); 3126 for (queue_idx = 0; queue_idx < group->queue_count; queue_idx++) { 3127 struct panthor_queue *queue = group->queues[queue_idx]; 3128 3129 if (!queue) 3130 continue; 3131 3132 queue_check_job_completion(queue); 3133 } 3134 dma_fence_end_signalling(cookie); 3135 3136 group_put(group); 3137} 3138 3139struct panthor_job_ringbuf_instrs { 3140 u64 buffer[MAX_INSTRS_PER_JOB]; 3141 u32 count; 3142}; 3143 3144struct panthor_job_instr { 3145 u32 profile_mask; 3146 u64 instr; 3147}; 3148 3149#define JOB_INSTR(__prof, __instr) \ 3150 { \ 3151 .profile_mask = __prof, \ 3152 .instr = __instr, \ 3153 } 3154 3155static void 3156copy_instrs_to_ringbuf(struct panthor_queue *queue, 3157 struct panthor_job *job, 3158 struct panthor_job_ringbuf_instrs *instrs) 3159{ 3160 u64 ringbuf_size = panthor_kernel_bo_size(queue->ringbuf); 3161 u64 start = job->ringbuf.start & (ringbuf_size - 1); 3162 u64 size, written; 3163 3164 /* 3165 * We need to write a whole slot, including any trailing zeroes 3166 * that may come at the end of it. Also, because instrs.buffer has 3167 * been zero-initialised, there's no need to pad it with 0's 3168 */ 3169 instrs->count = ALIGN(instrs->count, NUM_INSTRS_PER_CACHE_LINE); 3170 size = instrs->count * sizeof(u64); 3171 WARN_ON(size > ringbuf_size); 3172 written = min(ringbuf_size - start, size); 3173 3174 memcpy(queue->ringbuf->kmap + start, instrs->buffer, written); 3175 3176 if (written < size) 3177 memcpy(queue->ringbuf->kmap, 3178 &instrs->buffer[written / sizeof(u64)], 3179 size - written); 3180} 3181 3182struct panthor_job_cs_params { 3183 u32 profile_mask; 3184 u64 addr_reg; u64 val_reg; 3185 u64 cycle_reg; u64 time_reg; 3186 u64 sync_addr; u64 times_addr; 3187 u64 cs_start; u64 cs_size; 3188 u32 last_flush; u32 waitall_mask; 3189}; 3190 3191static void 3192get_job_cs_params(struct panthor_job *job, struct panthor_job_cs_params *params) 3193{ 3194 struct panthor_group *group = job->group; 3195 struct panthor_queue *queue = group->queues[job->queue_idx]; 3196 struct panthor_device *ptdev = group->ptdev; 3197 struct panthor_scheduler *sched = ptdev->scheduler; 3198 3199 params->addr_reg = ptdev->csif_info.cs_reg_count - 3200 ptdev->csif_info.unpreserved_cs_reg_count; 3201 params->val_reg = params->addr_reg + 2; 3202 params->cycle_reg = params->addr_reg; 3203 params->time_reg = params->val_reg; 3204 3205 params->sync_addr = panthor_kernel_bo_gpuva(group->syncobjs) + 3206 job->queue_idx * sizeof(struct panthor_syncobj_64b); 3207 params->times_addr = panthor_kernel_bo_gpuva(queue->profiling.slots) + 3208 (job->profiling.slot * sizeof(struct panthor_job_profiling_data)); 3209 params->waitall_mask = GENMASK(sched->sb_slot_count - 1, 0); 3210 3211 params->cs_start = job->call_info.start; 3212 params->cs_size = job->call_info.size; 3213 params->last_flush = job->call_info.latest_flush; 3214 3215 params->profile_mask = job->profiling.mask; 3216} 3217 3218#define JOB_INSTR_ALWAYS(instr) \ 3219 JOB_INSTR(PANTHOR_DEVICE_PROFILING_DISABLED, (instr)) 3220#define JOB_INSTR_TIMESTAMP(instr) \ 3221 JOB_INSTR(PANTHOR_DEVICE_PROFILING_TIMESTAMP, (instr)) 3222#define JOB_INSTR_CYCLES(instr) \ 3223 JOB_INSTR(PANTHOR_DEVICE_PROFILING_CYCLES, (instr)) 3224 3225static void 3226prepare_job_instrs(const struct panthor_job_cs_params *params, 3227 struct panthor_job_ringbuf_instrs *instrs) 3228{ 3229 const struct panthor_job_instr instr_seq[] = { 3230 /* MOV32 rX+2, cs.latest_flush */ 3231 JOB_INSTR_ALWAYS((2ull << 56) | (params->val_reg << 48) | params->last_flush), 3232 /* FLUSH_CACHE2.clean_inv_all.no_wait.signal(0) rX+2 */ 3233 JOB_INSTR_ALWAYS((36ull << 56) | (0ull << 48) | (params->val_reg << 40) | 3234 (0 << 16) | 0x233), 3235 /* MOV48 rX:rX+1, cycles_offset */ 3236 JOB_INSTR_CYCLES((1ull << 56) | (params->cycle_reg << 48) | 3237 (params->times_addr + 3238 offsetof(struct panthor_job_profiling_data, cycles.before))), 3239 /* STORE_STATE cycles */ 3240 JOB_INSTR_CYCLES((40ull << 56) | (params->cycle_reg << 40) | (1ll << 32)), 3241 /* MOV48 rX:rX+1, time_offset */ 3242 JOB_INSTR_TIMESTAMP((1ull << 56) | (params->time_reg << 48) | 3243 (params->times_addr + 3244 offsetof(struct panthor_job_profiling_data, time.before))), 3245 /* STORE_STATE timer */ 3246 JOB_INSTR_TIMESTAMP((40ull << 56) | (params->time_reg << 40) | (0ll << 32)), 3247 /* MOV48 rX:rX+1, cs.start */ 3248 JOB_INSTR_ALWAYS((1ull << 56) | (params->addr_reg << 48) | params->cs_start), 3249 /* MOV32 rX+2, cs.size */ 3250 JOB_INSTR_ALWAYS((2ull << 56) | (params->val_reg << 48) | params->cs_size), 3251 /* WAIT(0) => waits for FLUSH_CACHE2 instruction */ 3252 JOB_INSTR_ALWAYS((3ull << 56) | (1 << 16)), 3253 /* CALL rX:rX+1, rX+2 */ 3254 JOB_INSTR_ALWAYS((32ull << 56) | (params->addr_reg << 40) | 3255 (params->val_reg << 32)), 3256 /* MOV48 rX:rX+1, cycles_offset */ 3257 JOB_INSTR_CYCLES((1ull << 56) | (params->cycle_reg << 48) | 3258 (params->times_addr + 3259 offsetof(struct panthor_job_profiling_data, cycles.after))), 3260 /* STORE_STATE cycles */ 3261 JOB_INSTR_CYCLES((40ull << 56) | (params->cycle_reg << 40) | (1ll << 32)), 3262 /* MOV48 rX:rX+1, time_offset */ 3263 JOB_INSTR_TIMESTAMP((1ull << 56) | (params->time_reg << 48) | 3264 (params->times_addr + 3265 offsetof(struct panthor_job_profiling_data, time.after))), 3266 /* STORE_STATE timer */ 3267 JOB_INSTR_TIMESTAMP((40ull << 56) | (params->time_reg << 40) | (0ll << 32)), 3268 /* MOV48 rX:rX+1, sync_addr */ 3269 JOB_INSTR_ALWAYS((1ull << 56) | (params->addr_reg << 48) | params->sync_addr), 3270 /* MOV48 rX+2, #1 */ 3271 JOB_INSTR_ALWAYS((1ull << 56) | (params->val_reg << 48) | 1), 3272 /* WAIT(all) */ 3273 JOB_INSTR_ALWAYS((3ull << 56) | (params->waitall_mask << 16)), 3274 /* SYNC_ADD64.system_scope.propage_err.nowait rX:rX+1, rX+2*/ 3275 JOB_INSTR_ALWAYS((51ull << 56) | (0ull << 48) | (params->addr_reg << 40) | 3276 (params->val_reg << 32) | (0 << 16) | 1), 3277 /* ERROR_BARRIER, so we can recover from faults at job boundaries. */ 3278 JOB_INSTR_ALWAYS((47ull << 56)), 3279 }; 3280 u32 pad; 3281 3282 instrs->count = 0; 3283 3284 /* NEED to be cacheline aligned to please the prefetcher. */ 3285 static_assert(sizeof(instrs->buffer) % 64 == 0, 3286 "panthor_job_ringbuf_instrs::buffer is not aligned on a cacheline"); 3287 3288 /* Make sure we have enough storage to store the whole sequence. */ 3289 static_assert(ALIGN(ARRAY_SIZE(instr_seq), NUM_INSTRS_PER_CACHE_LINE) == 3290 ARRAY_SIZE(instrs->buffer), 3291 "instr_seq vs panthor_job_ringbuf_instrs::buffer size mismatch"); 3292 3293 for (u32 i = 0; i < ARRAY_SIZE(instr_seq); i++) { 3294 /* If the profile mask of this instruction is not enabled, skip it. */ 3295 if (instr_seq[i].profile_mask && 3296 !(instr_seq[i].profile_mask & params->profile_mask)) 3297 continue; 3298 3299 instrs->buffer[instrs->count++] = instr_seq[i].instr; 3300 } 3301 3302 pad = ALIGN(instrs->count, NUM_INSTRS_PER_CACHE_LINE); 3303 memset(&instrs->buffer[instrs->count], 0, 3304 (pad - instrs->count) * sizeof(instrs->buffer[0])); 3305 instrs->count = pad; 3306} 3307 3308static u32 calc_job_credits(u32 profile_mask) 3309{ 3310 struct panthor_job_ringbuf_instrs instrs; 3311 struct panthor_job_cs_params params = { 3312 .profile_mask = profile_mask, 3313 }; 3314 3315 prepare_job_instrs(¶ms, &instrs); 3316 return instrs.count; 3317} 3318 3319static struct dma_fence * 3320queue_run_job(struct drm_sched_job *sched_job) 3321{ 3322 struct panthor_job *job = container_of(sched_job, struct panthor_job, base); 3323 struct panthor_group *group = job->group; 3324 struct panthor_queue *queue = group->queues[job->queue_idx]; 3325 struct panthor_device *ptdev = group->ptdev; 3326 struct panthor_scheduler *sched = ptdev->scheduler; 3327 struct panthor_job_ringbuf_instrs instrs; 3328 struct panthor_job_cs_params cs_params; 3329 struct dma_fence *done_fence; 3330 int ret; 3331 3332 /* Stream size is zero, nothing to do except making sure all previously 3333 * submitted jobs are done before we signal the 3334 * drm_sched_job::s_fence::finished fence. 3335 */ 3336 if (!job->call_info.size) { 3337 job->done_fence = dma_fence_get(queue->fence_ctx.last_fence); 3338 return dma_fence_get(job->done_fence); 3339 } 3340 3341 ret = panthor_device_resume_and_get(ptdev); 3342 if (drm_WARN_ON(&ptdev->base, ret)) 3343 return ERR_PTR(ret); 3344 3345 mutex_lock(&sched->lock); 3346 if (!group_can_run(group)) { 3347 done_fence = ERR_PTR(-ECANCELED); 3348 goto out_unlock; 3349 } 3350 3351 dma_fence_init(job->done_fence, 3352 &panthor_queue_fence_ops, 3353 &queue->fence_ctx.lock, 3354 queue->fence_ctx.id, 3355 atomic64_inc_return(&queue->fence_ctx.seqno)); 3356 3357 job->profiling.slot = queue->profiling.seqno++; 3358 if (queue->profiling.seqno == queue->profiling.slot_count) 3359 queue->profiling.seqno = 0; 3360 3361 job->ringbuf.start = queue->iface.input->insert; 3362 3363 get_job_cs_params(job, &cs_params); 3364 prepare_job_instrs(&cs_params, &instrs); 3365 copy_instrs_to_ringbuf(queue, job, &instrs); 3366 3367 job->ringbuf.end = job->ringbuf.start + (instrs.count * sizeof(u64)); 3368 3369 panthor_job_get(&job->base); 3370 spin_lock(&queue->fence_ctx.lock); 3371 list_add_tail(&job->node, &queue->fence_ctx.in_flight_jobs); 3372 spin_unlock(&queue->fence_ctx.lock); 3373 3374 /* Make sure the ring buffer is updated before the INSERT 3375 * register. 3376 */ 3377 wmb(); 3378 3379 queue->iface.input->extract = queue->iface.output->extract; 3380 queue->iface.input->insert = job->ringbuf.end; 3381 3382 if (group->csg_id < 0) { 3383 group_schedule_locked(group, BIT(job->queue_idx)); 3384 } else { 3385 u32 queue_mask = BIT(job->queue_idx); 3386 bool resume_tick = group_is_idle(group) && 3387 (group->idle_queues & queue_mask) && 3388 !(group->blocked_queues & queue_mask) && 3389 sched->resched_target == U64_MAX; 3390 3391 /* We just added something to the queue, so it's no longer idle. */ 3392 group->idle_queues &= ~queue_mask; 3393 3394 if (resume_tick) 3395 sched_resume_tick(ptdev); 3396 3397 gpu_write(ptdev, CSF_DOORBELL(queue->doorbell_id), 1); 3398 if (!sched->pm.has_ref && 3399 !(group->blocked_queues & BIT(job->queue_idx))) { 3400 pm_runtime_get(ptdev->base.dev); 3401 sched->pm.has_ref = true; 3402 } 3403 queue_resume_timeout(queue); 3404 panthor_devfreq_record_busy(sched->ptdev); 3405 } 3406 3407 /* Update the last fence. */ 3408 dma_fence_put(queue->fence_ctx.last_fence); 3409 queue->fence_ctx.last_fence = dma_fence_get(job->done_fence); 3410 3411 done_fence = dma_fence_get(job->done_fence); 3412 3413out_unlock: 3414 mutex_unlock(&sched->lock); 3415 pm_runtime_mark_last_busy(ptdev->base.dev); 3416 pm_runtime_put_autosuspend(ptdev->base.dev); 3417 3418 return done_fence; 3419} 3420 3421static enum drm_gpu_sched_stat 3422queue_timedout_job(struct drm_sched_job *sched_job) 3423{ 3424 struct panthor_job *job = container_of(sched_job, struct panthor_job, base); 3425 struct panthor_group *group = job->group; 3426 struct panthor_device *ptdev = group->ptdev; 3427 struct panthor_scheduler *sched = ptdev->scheduler; 3428 struct panthor_queue *queue = group->queues[job->queue_idx]; 3429 3430 drm_warn(&ptdev->base, "job timeout: pid=%d, comm=%s, seqno=%llu\n", 3431 group->task_info.pid, group->task_info.comm, job->done_fence->seqno); 3432 3433 drm_WARN_ON(&ptdev->base, atomic_read(&sched->reset.in_progress)); 3434 3435 queue_stop(queue, job); 3436 3437 mutex_lock(&sched->lock); 3438 group->timedout = true; 3439 if (group->csg_id >= 0) { 3440 sched_queue_delayed_work(ptdev->scheduler, tick, 0); 3441 } else { 3442 /* Remove from the run queues, so the scheduler can't 3443 * pick the group on the next tick. 3444 */ 3445 list_del_init(&group->run_node); 3446 list_del_init(&group->wait_node); 3447 3448 group_queue_work(group, term); 3449 } 3450 mutex_unlock(&sched->lock); 3451 3452 queue_start(queue); 3453 return DRM_GPU_SCHED_STAT_RESET; 3454} 3455 3456static void queue_free_job(struct drm_sched_job *sched_job) 3457{ 3458 drm_sched_job_cleanup(sched_job); 3459 panthor_job_put(sched_job); 3460} 3461 3462static const struct drm_sched_backend_ops panthor_queue_sched_ops = { 3463 .run_job = queue_run_job, 3464 .timedout_job = queue_timedout_job, 3465 .free_job = queue_free_job, 3466}; 3467 3468static u32 calc_profiling_ringbuf_num_slots(struct panthor_device *ptdev, 3469 u32 cs_ringbuf_size) 3470{ 3471 u32 min_profiled_job_instrs = U32_MAX; 3472 u32 last_flag = fls(PANTHOR_DEVICE_PROFILING_ALL); 3473 3474 /* 3475 * We want to calculate the minimum size of a profiled job's CS, 3476 * because since they need additional instructions for the sampling 3477 * of performance metrics, they might take up further slots in 3478 * the queue's ringbuffer. This means we might not need as many job 3479 * slots for keeping track of their profiling information. What we 3480 * need is the maximum number of slots we should allocate to this end, 3481 * which matches the maximum number of profiled jobs we can place 3482 * simultaneously in the queue's ring buffer. 3483 * That has to be calculated separately for every single job profiling 3484 * flag, but not in the case job profiling is disabled, since unprofiled 3485 * jobs don't need to keep track of this at all. 3486 */ 3487 for (u32 i = 0; i < last_flag; i++) { 3488 min_profiled_job_instrs = 3489 min(min_profiled_job_instrs, calc_job_credits(BIT(i))); 3490 } 3491 3492 return DIV_ROUND_UP(cs_ringbuf_size, min_profiled_job_instrs * sizeof(u64)); 3493} 3494 3495static void queue_timeout_work(struct work_struct *work) 3496{ 3497 struct panthor_queue *queue = container_of(work, struct panthor_queue, 3498 timeout.work.work); 3499 bool progress; 3500 3501 progress = queue_check_job_completion(queue); 3502 if (!progress) 3503 drm_sched_fault(&queue->scheduler); 3504} 3505 3506static struct panthor_queue * 3507group_create_queue(struct panthor_group *group, 3508 const struct drm_panthor_queue_create *args, 3509 u64 drm_client_id, u32 gid, u32 qid) 3510{ 3511 struct drm_sched_init_args sched_args = { 3512 .ops = &panthor_queue_sched_ops, 3513 .submit_wq = group->ptdev->scheduler->wq, 3514 .num_rqs = 1, 3515 /* 3516 * The credit limit argument tells us the total number of 3517 * instructions across all CS slots in the ringbuffer, with 3518 * some jobs requiring twice as many as others, depending on 3519 * their profiling status. 3520 */ 3521 .credit_limit = args->ringbuf_size / sizeof(u64), 3522 .timeout = MAX_SCHEDULE_TIMEOUT, 3523 .timeout_wq = group->ptdev->reset.wq, 3524 .dev = group->ptdev->base.dev, 3525 }; 3526 struct drm_gpu_scheduler *drm_sched; 3527 struct panthor_queue *queue; 3528 int ret; 3529 3530 if (args->pad[0] || args->pad[1] || args->pad[2]) 3531 return ERR_PTR(-EINVAL); 3532 3533 if (args->ringbuf_size < SZ_4K || args->ringbuf_size > SZ_64K || 3534 !is_power_of_2(args->ringbuf_size)) 3535 return ERR_PTR(-EINVAL); 3536 3537 if (args->priority > CSF_MAX_QUEUE_PRIO) 3538 return ERR_PTR(-EINVAL); 3539 3540 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 3541 if (!queue) 3542 return ERR_PTR(-ENOMEM); 3543 3544 queue->timeout.remaining = msecs_to_jiffies(JOB_TIMEOUT_MS); 3545 INIT_DELAYED_WORK(&queue->timeout.work, queue_timeout_work); 3546 queue->fence_ctx.id = dma_fence_context_alloc(1); 3547 spin_lock_init(&queue->fence_ctx.lock); 3548 INIT_LIST_HEAD(&queue->fence_ctx.in_flight_jobs); 3549 3550 queue->priority = args->priority; 3551 3552 queue->ringbuf = panthor_kernel_bo_create(group->ptdev, group->vm, 3553 args->ringbuf_size, 3554 DRM_PANTHOR_BO_NO_MMAP, 3555 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | 3556 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED, 3557 PANTHOR_VM_KERNEL_AUTO_VA, 3558 "CS ring buffer"); 3559 if (IS_ERR(queue->ringbuf)) { 3560 ret = PTR_ERR(queue->ringbuf); 3561 goto err_free_queue; 3562 } 3563 3564 ret = panthor_kernel_bo_vmap(queue->ringbuf); 3565 if (ret) 3566 goto err_free_queue; 3567 3568 queue->iface.mem = panthor_fw_alloc_queue_iface_mem(group->ptdev, 3569 &queue->iface.input, 3570 &queue->iface.output, 3571 &queue->iface.input_fw_va, 3572 &queue->iface.output_fw_va); 3573 if (IS_ERR(queue->iface.mem)) { 3574 ret = PTR_ERR(queue->iface.mem); 3575 goto err_free_queue; 3576 } 3577 3578 queue->profiling.slot_count = 3579 calc_profiling_ringbuf_num_slots(group->ptdev, args->ringbuf_size); 3580 3581 queue->profiling.slots = 3582 panthor_kernel_bo_create(group->ptdev, group->vm, 3583 queue->profiling.slot_count * 3584 sizeof(struct panthor_job_profiling_data), 3585 DRM_PANTHOR_BO_NO_MMAP, 3586 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | 3587 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED, 3588 PANTHOR_VM_KERNEL_AUTO_VA, 3589 "Group job stats"); 3590 3591 if (IS_ERR(queue->profiling.slots)) { 3592 ret = PTR_ERR(queue->profiling.slots); 3593 goto err_free_queue; 3594 } 3595 3596 ret = panthor_kernel_bo_vmap(queue->profiling.slots); 3597 if (ret) 3598 goto err_free_queue; 3599 3600 /* assign a unique name */ 3601 queue->name = kasprintf(GFP_KERNEL, "panthor-queue-%llu-%u-%u", drm_client_id, gid, qid); 3602 if (!queue->name) { 3603 ret = -ENOMEM; 3604 goto err_free_queue; 3605 } 3606 3607 sched_args.name = queue->name; 3608 3609 ret = drm_sched_init(&queue->scheduler, &sched_args); 3610 if (ret) 3611 goto err_free_queue; 3612 3613 drm_sched = &queue->scheduler; 3614 ret = drm_sched_entity_init(&queue->entity, 0, &drm_sched, 1, NULL); 3615 if (ret) 3616 goto err_free_queue; 3617 3618 return queue; 3619 3620err_free_queue: 3621 group_free_queue(group, queue); 3622 return ERR_PTR(ret); 3623} 3624 3625static void group_init_task_info(struct panthor_group *group) 3626{ 3627 struct task_struct *task = current->group_leader; 3628 3629 group->task_info.pid = task->pid; 3630 get_task_comm(group->task_info.comm, task); 3631} 3632 3633static void add_group_kbo_sizes(struct panthor_device *ptdev, 3634 struct panthor_group *group) 3635{ 3636 struct panthor_queue *queue; 3637 int i; 3638 3639 if (drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(group))) 3640 return; 3641 if (drm_WARN_ON(&ptdev->base, ptdev != group->ptdev)) 3642 return; 3643 3644 group->fdinfo.kbo_sizes += group->suspend_buf->obj->size; 3645 group->fdinfo.kbo_sizes += group->protm_suspend_buf->obj->size; 3646 group->fdinfo.kbo_sizes += group->syncobjs->obj->size; 3647 3648 for (i = 0; i < group->queue_count; i++) { 3649 queue = group->queues[i]; 3650 group->fdinfo.kbo_sizes += queue->ringbuf->obj->size; 3651 group->fdinfo.kbo_sizes += queue->iface.mem->obj->size; 3652 group->fdinfo.kbo_sizes += queue->profiling.slots->obj->size; 3653 } 3654} 3655 3656#define MAX_GROUPS_PER_POOL 128 3657 3658int panthor_group_create(struct panthor_file *pfile, 3659 const struct drm_panthor_group_create *group_args, 3660 const struct drm_panthor_queue_create *queue_args, 3661 u64 drm_client_id) 3662{ 3663 struct panthor_device *ptdev = pfile->ptdev; 3664 struct panthor_group_pool *gpool = pfile->groups; 3665 struct panthor_scheduler *sched = ptdev->scheduler; 3666 struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, 0); 3667 struct panthor_group *group = NULL; 3668 u32 gid, i, suspend_size; 3669 int ret; 3670 3671 if (group_args->pad) 3672 return -EINVAL; 3673 3674 if (group_args->priority >= PANTHOR_CSG_PRIORITY_COUNT) 3675 return -EINVAL; 3676 3677 if ((group_args->compute_core_mask & ~ptdev->gpu_info.shader_present) || 3678 (group_args->fragment_core_mask & ~ptdev->gpu_info.shader_present) || 3679 (group_args->tiler_core_mask & ~ptdev->gpu_info.tiler_present)) 3680 return -EINVAL; 3681 3682 if (hweight64(group_args->compute_core_mask) < group_args->max_compute_cores || 3683 hweight64(group_args->fragment_core_mask) < group_args->max_fragment_cores || 3684 hweight64(group_args->tiler_core_mask) < group_args->max_tiler_cores) 3685 return -EINVAL; 3686 3687 group = kzalloc(sizeof(*group), GFP_KERNEL); 3688 if (!group) 3689 return -ENOMEM; 3690 3691 spin_lock_init(&group->fatal_lock); 3692 kref_init(&group->refcount); 3693 group->state = PANTHOR_CS_GROUP_CREATED; 3694 group->csg_id = -1; 3695 3696 group->ptdev = ptdev; 3697 group->max_compute_cores = group_args->max_compute_cores; 3698 group->compute_core_mask = group_args->compute_core_mask; 3699 group->max_fragment_cores = group_args->max_fragment_cores; 3700 group->fragment_core_mask = group_args->fragment_core_mask; 3701 group->max_tiler_cores = group_args->max_tiler_cores; 3702 group->tiler_core_mask = group_args->tiler_core_mask; 3703 group->priority = group_args->priority; 3704 3705 INIT_LIST_HEAD(&group->wait_node); 3706 INIT_LIST_HEAD(&group->run_node); 3707 INIT_WORK(&group->term_work, group_term_work); 3708 INIT_WORK(&group->sync_upd_work, group_sync_upd_work); 3709 INIT_WORK(&group->tiler_oom_work, group_tiler_oom_work); 3710 INIT_WORK(&group->release_work, group_release_work); 3711 3712 group->vm = panthor_vm_pool_get_vm(pfile->vms, group_args->vm_id); 3713 if (!group->vm) { 3714 ret = -EINVAL; 3715 goto err_put_group; 3716 } 3717 3718 suspend_size = csg_iface->control->suspend_size; 3719 group->suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size); 3720 if (IS_ERR(group->suspend_buf)) { 3721 ret = PTR_ERR(group->suspend_buf); 3722 group->suspend_buf = NULL; 3723 goto err_put_group; 3724 } 3725 3726 suspend_size = csg_iface->control->protm_suspend_size; 3727 group->protm_suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size); 3728 if (IS_ERR(group->protm_suspend_buf)) { 3729 ret = PTR_ERR(group->protm_suspend_buf); 3730 group->protm_suspend_buf = NULL; 3731 goto err_put_group; 3732 } 3733 3734 group->syncobjs = panthor_kernel_bo_create(ptdev, group->vm, 3735 group_args->queues.count * 3736 sizeof(struct panthor_syncobj_64b), 3737 DRM_PANTHOR_BO_NO_MMAP, 3738 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | 3739 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED, 3740 PANTHOR_VM_KERNEL_AUTO_VA, 3741 "Group sync objects"); 3742 if (IS_ERR(group->syncobjs)) { 3743 ret = PTR_ERR(group->syncobjs); 3744 goto err_put_group; 3745 } 3746 3747 ret = panthor_kernel_bo_vmap(group->syncobjs); 3748 if (ret) 3749 goto err_put_group; 3750 3751 memset(group->syncobjs->kmap, 0, 3752 group_args->queues.count * sizeof(struct panthor_syncobj_64b)); 3753 3754 ret = xa_alloc(&gpool->xa, &gid, group, XA_LIMIT(1, MAX_GROUPS_PER_POOL), GFP_KERNEL); 3755 if (ret) 3756 goto err_put_group; 3757 3758 for (i = 0; i < group_args->queues.count; i++) { 3759 group->queues[i] = group_create_queue(group, &queue_args[i], drm_client_id, gid, i); 3760 if (IS_ERR(group->queues[i])) { 3761 ret = PTR_ERR(group->queues[i]); 3762 group->queues[i] = NULL; 3763 goto err_erase_gid; 3764 } 3765 3766 group->queue_count++; 3767 } 3768 3769 group->idle_queues = GENMASK(group->queue_count - 1, 0); 3770 3771 mutex_lock(&sched->reset.lock); 3772 if (atomic_read(&sched->reset.in_progress)) { 3773 panthor_group_stop(group); 3774 } else { 3775 mutex_lock(&sched->lock); 3776 list_add_tail(&group->run_node, 3777 &sched->groups.idle[group->priority]); 3778 mutex_unlock(&sched->lock); 3779 } 3780 mutex_unlock(&sched->reset.lock); 3781 3782 add_group_kbo_sizes(group->ptdev, group); 3783 spin_lock_init(&group->fdinfo.lock); 3784 3785 group_init_task_info(group); 3786 3787 xa_set_mark(&gpool->xa, gid, GROUP_REGISTERED); 3788 3789 return gid; 3790 3791err_erase_gid: 3792 xa_erase(&gpool->xa, gid); 3793 3794err_put_group: 3795 group_put(group); 3796 return ret; 3797} 3798 3799int panthor_group_destroy(struct panthor_file *pfile, u32 group_handle) 3800{ 3801 struct panthor_group_pool *gpool = pfile->groups; 3802 struct panthor_device *ptdev = pfile->ptdev; 3803 struct panthor_scheduler *sched = ptdev->scheduler; 3804 struct panthor_group *group; 3805 3806 if (!xa_get_mark(&gpool->xa, group_handle, GROUP_REGISTERED)) 3807 return -EINVAL; 3808 3809 group = xa_erase(&gpool->xa, group_handle); 3810 if (!group) 3811 return -EINVAL; 3812 3813 mutex_lock(&sched->reset.lock); 3814 mutex_lock(&sched->lock); 3815 group->destroyed = true; 3816 if (group->csg_id >= 0) { 3817 sched_queue_delayed_work(sched, tick, 0); 3818 } else if (!atomic_read(&sched->reset.in_progress)) { 3819 /* Remove from the run queues, so the scheduler can't 3820 * pick the group on the next tick. 3821 */ 3822 list_del_init(&group->run_node); 3823 list_del_init(&group->wait_node); 3824 group_queue_work(group, term); 3825 } 3826 mutex_unlock(&sched->lock); 3827 mutex_unlock(&sched->reset.lock); 3828 3829 group_put(group); 3830 return 0; 3831} 3832 3833static struct panthor_group *group_from_handle(struct panthor_group_pool *pool, 3834 unsigned long group_handle) 3835{ 3836 struct panthor_group *group; 3837 3838 xa_lock(&pool->xa); 3839 group = group_get(xa_find(&pool->xa, &group_handle, group_handle, GROUP_REGISTERED)); 3840 xa_unlock(&pool->xa); 3841 3842 return group; 3843} 3844 3845int panthor_group_get_state(struct panthor_file *pfile, 3846 struct drm_panthor_group_get_state *get_state) 3847{ 3848 struct panthor_group_pool *gpool = pfile->groups; 3849 struct panthor_device *ptdev = pfile->ptdev; 3850 struct panthor_scheduler *sched = ptdev->scheduler; 3851 struct panthor_group *group; 3852 3853 if (get_state->pad) 3854 return -EINVAL; 3855 3856 group = group_from_handle(gpool, get_state->group_handle); 3857 if (!group) 3858 return -EINVAL; 3859 3860 memset(get_state, 0, sizeof(*get_state)); 3861 3862 mutex_lock(&sched->lock); 3863 if (group->timedout) 3864 get_state->state |= DRM_PANTHOR_GROUP_STATE_TIMEDOUT; 3865 if (group->fatal_queues) { 3866 get_state->state |= DRM_PANTHOR_GROUP_STATE_FATAL_FAULT; 3867 get_state->fatal_queues = group->fatal_queues; 3868 } 3869 if (group->innocent) 3870 get_state->state |= DRM_PANTHOR_GROUP_STATE_INNOCENT; 3871 mutex_unlock(&sched->lock); 3872 3873 group_put(group); 3874 return 0; 3875} 3876 3877int panthor_group_pool_create(struct panthor_file *pfile) 3878{ 3879 struct panthor_group_pool *gpool; 3880 3881 gpool = kzalloc(sizeof(*gpool), GFP_KERNEL); 3882 if (!gpool) 3883 return -ENOMEM; 3884 3885 xa_init_flags(&gpool->xa, XA_FLAGS_ALLOC1); 3886 pfile->groups = gpool; 3887 return 0; 3888} 3889 3890void panthor_group_pool_destroy(struct panthor_file *pfile) 3891{ 3892 struct panthor_group_pool *gpool = pfile->groups; 3893 struct panthor_group *group; 3894 unsigned long i; 3895 3896 if (IS_ERR_OR_NULL(gpool)) 3897 return; 3898 3899 xa_for_each(&gpool->xa, i, group) 3900 panthor_group_destroy(pfile, i); 3901 3902 xa_destroy(&gpool->xa); 3903 kfree(gpool); 3904 pfile->groups = NULL; 3905} 3906 3907/** 3908 * panthor_fdinfo_gather_group_mem_info() - Retrieve aggregate size of all private kernel BO's 3909 * belonging to all the groups owned by an open Panthor file 3910 * @pfile: File. 3911 * @stats: Memory statistics to be updated. 3912 * 3913 */ 3914void 3915panthor_fdinfo_gather_group_mem_info(struct panthor_file *pfile, 3916 struct drm_memory_stats *stats) 3917{ 3918 struct panthor_group_pool *gpool = pfile->groups; 3919 struct panthor_group *group; 3920 unsigned long i; 3921 3922 if (IS_ERR_OR_NULL(gpool)) 3923 return; 3924 3925 xa_lock(&gpool->xa); 3926 xa_for_each_marked(&gpool->xa, i, group, GROUP_REGISTERED) { 3927 stats->resident += group->fdinfo.kbo_sizes; 3928 if (group->csg_id >= 0) 3929 stats->active += group->fdinfo.kbo_sizes; 3930 } 3931 xa_unlock(&gpool->xa); 3932} 3933 3934static void job_release(struct kref *ref) 3935{ 3936 struct panthor_job *job = container_of(ref, struct panthor_job, refcount); 3937 3938 drm_WARN_ON(&job->group->ptdev->base, !list_empty(&job->node)); 3939 3940 if (job->base.s_fence) 3941 drm_sched_job_cleanup(&job->base); 3942 3943 if (job->done_fence && job->done_fence->ops) 3944 dma_fence_put(job->done_fence); 3945 else 3946 dma_fence_free(job->done_fence); 3947 3948 group_put(job->group); 3949 3950 kfree(job); 3951} 3952 3953struct drm_sched_job *panthor_job_get(struct drm_sched_job *sched_job) 3954{ 3955 if (sched_job) { 3956 struct panthor_job *job = container_of(sched_job, struct panthor_job, base); 3957 3958 kref_get(&job->refcount); 3959 } 3960 3961 return sched_job; 3962} 3963 3964void panthor_job_put(struct drm_sched_job *sched_job) 3965{ 3966 struct panthor_job *job = container_of(sched_job, struct panthor_job, base); 3967 3968 if (sched_job) 3969 kref_put(&job->refcount, job_release); 3970} 3971 3972struct panthor_vm *panthor_job_vm(struct drm_sched_job *sched_job) 3973{ 3974 struct panthor_job *job = container_of(sched_job, struct panthor_job, base); 3975 3976 return job->group->vm; 3977} 3978 3979struct drm_sched_job * 3980panthor_job_create(struct panthor_file *pfile, 3981 u16 group_handle, 3982 const struct drm_panthor_queue_submit *qsubmit, 3983 u64 drm_client_id) 3984{ 3985 struct panthor_group_pool *gpool = pfile->groups; 3986 struct panthor_job *job; 3987 u32 credits; 3988 int ret; 3989 3990 if (qsubmit->pad) 3991 return ERR_PTR(-EINVAL); 3992 3993 /* If stream_addr is zero, so stream_size should be. */ 3994 if ((qsubmit->stream_size == 0) != (qsubmit->stream_addr == 0)) 3995 return ERR_PTR(-EINVAL); 3996 3997 /* Make sure the address is aligned on 64-byte (cacheline) and the size is 3998 * aligned on 8-byte (instruction size). 3999 */ 4000 if ((qsubmit->stream_addr & 63) || (qsubmit->stream_size & 7)) 4001 return ERR_PTR(-EINVAL); 4002 4003 /* bits 24:30 must be zero. */ 4004 if (qsubmit->latest_flush & GENMASK(30, 24)) 4005 return ERR_PTR(-EINVAL); 4006 4007 job = kzalloc(sizeof(*job), GFP_KERNEL); 4008 if (!job) 4009 return ERR_PTR(-ENOMEM); 4010 4011 kref_init(&job->refcount); 4012 job->queue_idx = qsubmit->queue_index; 4013 job->call_info.size = qsubmit->stream_size; 4014 job->call_info.start = qsubmit->stream_addr; 4015 job->call_info.latest_flush = qsubmit->latest_flush; 4016 INIT_LIST_HEAD(&job->node); 4017 4018 job->group = group_from_handle(gpool, group_handle); 4019 if (!job->group) { 4020 ret = -EINVAL; 4021 goto err_put_job; 4022 } 4023 4024 if (!group_can_run(job->group)) { 4025 ret = -EINVAL; 4026 goto err_put_job; 4027 } 4028 4029 if (job->queue_idx >= job->group->queue_count || 4030 !job->group->queues[job->queue_idx]) { 4031 ret = -EINVAL; 4032 goto err_put_job; 4033 } 4034 4035 /* Empty command streams don't need a fence, they'll pick the one from 4036 * the previously submitted job. 4037 */ 4038 if (job->call_info.size) { 4039 job->done_fence = kzalloc(sizeof(*job->done_fence), GFP_KERNEL); 4040 if (!job->done_fence) { 4041 ret = -ENOMEM; 4042 goto err_put_job; 4043 } 4044 } 4045 4046 job->profiling.mask = pfile->ptdev->profile_mask; 4047 credits = calc_job_credits(job->profiling.mask); 4048 if (credits == 0) { 4049 ret = -EINVAL; 4050 goto err_put_job; 4051 } 4052 4053 ret = drm_sched_job_init(&job->base, 4054 &job->group->queues[job->queue_idx]->entity, 4055 credits, job->group, drm_client_id); 4056 if (ret) 4057 goto err_put_job; 4058 4059 return &job->base; 4060 4061err_put_job: 4062 panthor_job_put(&job->base); 4063 return ERR_PTR(ret); 4064} 4065 4066void panthor_job_update_resvs(struct drm_exec *exec, struct drm_sched_job *sched_job) 4067{ 4068 struct panthor_job *job = container_of(sched_job, struct panthor_job, base); 4069 4070 panthor_vm_update_resvs(job->group->vm, exec, &sched_job->s_fence->finished, 4071 DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP); 4072} 4073 4074void panthor_sched_unplug(struct panthor_device *ptdev) 4075{ 4076 struct panthor_scheduler *sched = ptdev->scheduler; 4077 4078 disable_delayed_work_sync(&sched->tick_work); 4079 disable_work_sync(&sched->fw_events_work); 4080 disable_work_sync(&sched->sync_upd_work); 4081 4082 mutex_lock(&sched->lock); 4083 if (sched->pm.has_ref) { 4084 pm_runtime_put(ptdev->base.dev); 4085 sched->pm.has_ref = false; 4086 } 4087 mutex_unlock(&sched->lock); 4088} 4089 4090static void panthor_sched_fini(struct drm_device *ddev, void *res) 4091{ 4092 struct panthor_scheduler *sched = res; 4093 int prio; 4094 4095 if (!sched || !sched->csg_slot_count) 4096 return; 4097 4098 if (sched->wq) 4099 destroy_workqueue(sched->wq); 4100 4101 if (sched->heap_alloc_wq) 4102 destroy_workqueue(sched->heap_alloc_wq); 4103 4104 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { 4105 drm_WARN_ON(ddev, !list_empty(&sched->groups.runnable[prio])); 4106 drm_WARN_ON(ddev, !list_empty(&sched->groups.idle[prio])); 4107 } 4108 4109 drm_WARN_ON(ddev, !list_empty(&sched->groups.waiting)); 4110} 4111 4112int panthor_sched_init(struct panthor_device *ptdev) 4113{ 4114 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); 4115 struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, 0); 4116 struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, 0, 0); 4117 struct panthor_scheduler *sched; 4118 u32 gpu_as_count, num_groups; 4119 int prio, ret; 4120 4121 sched = drmm_kzalloc(&ptdev->base, sizeof(*sched), GFP_KERNEL); 4122 if (!sched) 4123 return -ENOMEM; 4124 4125 /* The highest bit in JOB_INT_* is reserved for globabl IRQs. That 4126 * leaves 31 bits for CSG IRQs, hence the MAX_CSGS clamp here. 4127 */ 4128 num_groups = min_t(u32, MAX_CSGS, glb_iface->control->group_num); 4129 4130 /* The FW-side scheduler might deadlock if two groups with the same 4131 * priority try to access a set of resources that overlaps, with part 4132 * of the resources being allocated to one group and the other part to 4133 * the other group, both groups waiting for the remaining resources to 4134 * be allocated. To avoid that, it is recommended to assign each CSG a 4135 * different priority. In theory we could allow several groups to have 4136 * the same CSG priority if they don't request the same resources, but 4137 * that makes the scheduling logic more complicated, so let's clamp 4138 * the number of CSG slots to MAX_CSG_PRIO + 1 for now. 4139 */ 4140 num_groups = min_t(u32, MAX_CSG_PRIO + 1, num_groups); 4141 4142 /* We need at least one AS for the MCU and one for the GPU contexts. */ 4143 gpu_as_count = hweight32(ptdev->gpu_info.as_present & GENMASK(31, 1)); 4144 if (!gpu_as_count) { 4145 drm_err(&ptdev->base, "Not enough AS (%d, expected at least 2)", 4146 gpu_as_count + 1); 4147 return -EINVAL; 4148 } 4149 4150 sched->ptdev = ptdev; 4151 sched->sb_slot_count = CS_FEATURES_SCOREBOARDS(cs_iface->control->features); 4152 sched->csg_slot_count = num_groups; 4153 sched->cs_slot_count = csg_iface->control->stream_num; 4154 sched->as_slot_count = gpu_as_count; 4155 ptdev->csif_info.csg_slot_count = sched->csg_slot_count; 4156 ptdev->csif_info.cs_slot_count = sched->cs_slot_count; 4157 ptdev->csif_info.scoreboard_slot_count = sched->sb_slot_count; 4158 4159 sched->last_tick = 0; 4160 sched->resched_target = U64_MAX; 4161 sched->tick_period = msecs_to_jiffies(10); 4162 INIT_DELAYED_WORK(&sched->tick_work, tick_work); 4163 INIT_WORK(&sched->sync_upd_work, sync_upd_work); 4164 INIT_WORK(&sched->fw_events_work, process_fw_events_work); 4165 4166 ret = drmm_mutex_init(&ptdev->base, &sched->lock); 4167 if (ret) 4168 return ret; 4169 4170 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { 4171 INIT_LIST_HEAD(&sched->groups.runnable[prio]); 4172 INIT_LIST_HEAD(&sched->groups.idle[prio]); 4173 } 4174 INIT_LIST_HEAD(&sched->groups.waiting); 4175 4176 ret = drmm_mutex_init(&ptdev->base, &sched->reset.lock); 4177 if (ret) 4178 return ret; 4179 4180 INIT_LIST_HEAD(&sched->reset.stopped_groups); 4181 4182 /* sched->heap_alloc_wq will be used for heap chunk allocation on 4183 * tiler OOM events, which means we can't use the same workqueue for 4184 * the scheduler because works queued by the scheduler are in 4185 * the dma-signalling path. Allocate a dedicated heap_alloc_wq to 4186 * work around this limitation. 4187 * 4188 * FIXME: Ultimately, what we need is a failable/non-blocking GEM 4189 * allocation path that we can call when a heap OOM is reported. The 4190 * FW is smart enough to fall back on other methods if the kernel can't 4191 * allocate memory, and fail the tiling job if none of these 4192 * countermeasures worked. 4193 * 4194 * Set WQ_MEM_RECLAIM on sched->wq to unblock the situation when the 4195 * system is running out of memory. 4196 */ 4197 sched->heap_alloc_wq = alloc_workqueue("panthor-heap-alloc", WQ_UNBOUND, 0); 4198 sched->wq = alloc_workqueue("panthor-csf-sched", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); 4199 if (!sched->wq || !sched->heap_alloc_wq) { 4200 panthor_sched_fini(&ptdev->base, sched); 4201 drm_err(&ptdev->base, "Failed to allocate the workqueues"); 4202 return -ENOMEM; 4203 } 4204 4205 ret = drmm_add_action_or_reset(&ptdev->base, panthor_sched_fini, sched); 4206 if (ret) 4207 return ret; 4208 4209 ptdev->scheduler = sched; 4210 return 0; 4211}
~/
Read-only file preview.