xe_gt_idle.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2023 Intel Corporation
  4 */
  5
  6#include <drm/drm_managed.h>
  7
  8#include <generated/xe_wa_oob.h>
  9#include "xe_force_wake.h"
 10#include "xe_device.h"
 11#include "xe_gt.h"
 12#include "xe_gt_idle.h"
 13#include "xe_gt_sysfs.h"
 14#include "xe_guc_pc.h"
 15#include "regs/xe_gt_regs.h"
 16#include "xe_macros.h"
 17#include "xe_mmio.h"
 18#include "xe_pm.h"
 19#include "xe_sriov.h"
 20#include "xe_wa.h"
 21
 22/**
 23 * DOC: Xe GT Idle
 24 *
 25 * Contains functions that init GT idle features like C6
 26 *
 27 * device/gt#/gtidle/name - name of the state
 28 * device/gt#/gtidle/idle_residency_ms - Provides residency of the idle state in ms
 29 * device/gt#/gtidle/idle_status - Provides current idle state
 30 */
 31
 32static struct xe_gt_idle *dev_to_gtidle(struct device *dev)
 33{
 34	struct kobject *kobj = &dev->kobj;
 35
 36	return &kobj_to_gt(kobj->parent)->gtidle;
 37}
 38
 39static struct xe_gt *gtidle_to_gt(struct xe_gt_idle *gtidle)
 40{
 41	return container_of(gtidle, struct xe_gt, gtidle);
 42}
 43
 44static struct xe_guc_pc *gtidle_to_pc(struct xe_gt_idle *gtidle)
 45{
 46	return &gtidle_to_gt(gtidle)->uc.guc.pc;
 47}
 48
 49static struct xe_device *
 50pc_to_xe(struct xe_guc_pc *pc)
 51{
 52	struct xe_guc *guc = container_of(pc, struct xe_guc, pc);
 53	struct xe_gt *gt = container_of(guc, struct xe_gt, uc.guc);
 54
 55	return gt_to_xe(gt);
 56}
 57
 58static const char *gt_idle_state_to_string(enum xe_gt_idle_state state)
 59{
 60	switch (state) {
 61	case GT_IDLE_C0:
 62		return "gt-c0";
 63	case GT_IDLE_C6:
 64		return "gt-c6";
 65	default:
 66		return "unknown";
 67	}
 68}
 69
 70static u64 get_residency_ms(struct xe_gt_idle *gtidle, u64 cur_residency)
 71{
 72	u64 delta, overflow_residency, prev_residency;
 73
 74	lockdep_assert_held(&gtidle->lock);
 75
 76	overflow_residency = BIT_ULL(32);
 77
 78	/*
 79	 * Counter wrap handling
 80	 * Store previous hw counter values for counter wrap-around handling
 81	 * Relying on sufficient frequency of queries otherwise counters can still wrap.
 82	 */
 83	prev_residency = gtidle->prev_residency;
 84	gtidle->prev_residency = cur_residency;
 85
 86	/* delta */
 87	if (cur_residency >= prev_residency)
 88		delta = cur_residency - prev_residency;
 89	else
 90		delta = cur_residency + (overflow_residency - prev_residency);
 91
 92	/* Add delta to extended raw driver copy of idle residency */
 93	cur_residency = gtidle->cur_residency + delta;
 94	gtidle->cur_residency = cur_residency;
 95
 96	/* residency multiplier in ns, convert to ms */
 97	cur_residency = mul_u64_u32_div(cur_residency, gtidle->residency_multiplier, 1e6);
 98
 99	return cur_residency;
100}
101
102void xe_gt_idle_enable_pg(struct xe_gt *gt)
103{
104	struct xe_device *xe = gt_to_xe(gt);
105	struct xe_gt_idle *gtidle = &gt->gtidle;
106	struct xe_mmio *mmio = &gt->mmio;
107	u32 vcs_mask, vecs_mask;
108	unsigned int fw_ref;
109	int i, j;
110
111	if (IS_SRIOV_VF(xe))
112		return;
113
114	/* Disable CPG for PVC */
115	if (xe->info.platform == XE_PVC)
116		return;
117
118	xe_device_assert_mem_access(gt_to_xe(gt));
119
120	vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE);
121	vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE);
122
123	if (vcs_mask || vecs_mask)
124		gtidle->powergate_enable = MEDIA_POWERGATE_ENABLE;
125
126	if (xe_gt_is_main_type(gt))
127		gtidle->powergate_enable |= RENDER_POWERGATE_ENABLE;
128
129	if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255)
130		gtidle->powergate_enable |= MEDIA_SAMPLERS_POWERGATE_ENABLE;
131
132	if (xe->info.platform != XE_DG1) {
133		for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) {
134			if ((gt->info.engine_mask & BIT(i)))
135				gtidle->powergate_enable |= (VDN_HCP_POWERGATE_ENABLE(j) |
136							     VDN_MFXVDENC_POWERGATE_ENABLE(j));
137		}
138	}
139
140	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
141	if (xe->info.skip_guc_pc) {
142		/*
143		 * GuC sets the hysteresis value when GuC PC is enabled
144		 * else set it to 25 (25 * 1.28us)
145		 */
146		xe_mmio_write32(mmio, MEDIA_POWERGATE_IDLE_HYSTERESIS, 25);
147		xe_mmio_write32(mmio, RENDER_POWERGATE_IDLE_HYSTERESIS, 25);
148	}
149
150	if (XE_GT_WA(gt, 14020316580))
151		gtidle->powergate_enable &= ~(VDN_HCP_POWERGATE_ENABLE(0) |
152					      VDN_MFXVDENC_POWERGATE_ENABLE(0) |
153					      VDN_HCP_POWERGATE_ENABLE(2) |
154					      VDN_MFXVDENC_POWERGATE_ENABLE(2));
155
156	xe_mmio_write32(mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
157	xe_force_wake_put(gt_to_fw(gt), fw_ref);
158}
159
160void xe_gt_idle_disable_pg(struct xe_gt *gt)
161{
162	struct xe_gt_idle *gtidle = &gt->gtidle;
163	unsigned int fw_ref;
164
165	if (IS_SRIOV_VF(gt_to_xe(gt)))
166		return;
167
168	xe_device_assert_mem_access(gt_to_xe(gt));
169	gtidle->powergate_enable = 0;
170
171	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
172	xe_mmio_write32(&gt->mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
173	xe_force_wake_put(gt_to_fw(gt), fw_ref);
174}
175
176/**
177 * xe_gt_idle_pg_print - Xe powergating info
178 * @gt: GT object
179 * @p: drm_printer.
180 *
181 * This function prints the powergating information
182 *
183 * Return: 0 on success, negative error code otherwise
184 */
185int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p)
186{
187	struct xe_gt_idle *gtidle = &gt->gtidle;
188	struct xe_device *xe = gt_to_xe(gt);
189	enum xe_gt_idle_state state;
190	u32 pg_enabled, pg_status = 0;
191	u32 vcs_mask, vecs_mask;
192	unsigned int fw_ref;
193	int n;
194	/*
195	 * Media Slices
196	 *
197	 * Slice 0: VCS0, VCS1, VECS0
198	 * Slice 1: VCS2, VCS3, VECS1
199	 * Slice 2: VCS4, VCS5, VECS2
200	 * Slice 3: VCS6, VCS7, VECS3
201	 */
202	static const struct {
203		u64 engines;
204		u32 status_bit;
205	} media_slices[] = {
206		{(BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS1) |
207		  BIT(XE_HW_ENGINE_VECS0)), MEDIA_SLICE0_AWAKE_STATUS},
208
209		{(BIT(XE_HW_ENGINE_VCS2) | BIT(XE_HW_ENGINE_VCS3) |
210		   BIT(XE_HW_ENGINE_VECS1)), MEDIA_SLICE1_AWAKE_STATUS},
211
212		{(BIT(XE_HW_ENGINE_VCS4) | BIT(XE_HW_ENGINE_VCS5) |
213		   BIT(XE_HW_ENGINE_VECS2)), MEDIA_SLICE2_AWAKE_STATUS},
214
215		{(BIT(XE_HW_ENGINE_VCS6) | BIT(XE_HW_ENGINE_VCS7) |
216		   BIT(XE_HW_ENGINE_VECS3)), MEDIA_SLICE3_AWAKE_STATUS},
217	};
218
219	if (xe->info.platform == XE_PVC) {
220		drm_printf(p, "Power Gating not supported\n");
221		return 0;
222	}
223
224	state = gtidle->idle_status(gtidle_to_pc(gtidle));
225	pg_enabled = gtidle->powergate_enable;
226
227	/* Do not wake the GT to read powergating status */
228	if (state != GT_IDLE_C6) {
229		fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
230		if (!fw_ref)
231			return -ETIMEDOUT;
232
233		pg_enabled = xe_mmio_read32(&gt->mmio, POWERGATE_ENABLE);
234		pg_status = xe_mmio_read32(&gt->mmio, POWERGATE_DOMAIN_STATUS);
235
236		xe_force_wake_put(gt_to_fw(gt), fw_ref);
237	}
238
239	if (gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK) {
240		drm_printf(p, "Render Power Gating Enabled: %s\n",
241			   str_yes_no(pg_enabled & RENDER_POWERGATE_ENABLE));
242
243		drm_printf(p, "Render Power Gate Status: %s\n",
244			   str_up_down(pg_status & RENDER_AWAKE_STATUS));
245	}
246
247	vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE);
248	vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE);
249
250	/* Print media CPG status only if media is present */
251	if (vcs_mask || vecs_mask) {
252		drm_printf(p, "Media Power Gating Enabled: %s\n",
253			   str_yes_no(pg_enabled & MEDIA_POWERGATE_ENABLE));
254
255		for (n = 0; n < ARRAY_SIZE(media_slices); n++)
256			if (gt->info.engine_mask & media_slices[n].engines)
257				drm_printf(p, "Media Slice%d Power Gate Status: %s\n", n,
258					   str_up_down(pg_status & media_slices[n].status_bit));
259	}
260
261	if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255)
262		drm_printf(p, "Media Samplers Power Gating Enabled: %s\n",
263			   str_yes_no(pg_enabled & MEDIA_SAMPLERS_POWERGATE_ENABLE));
264
265	return 0;
266}
267
268static ssize_t name_show(struct kobject *kobj,
269			 struct kobj_attribute *attr, char *buff)
270{
271	struct device *dev = kobj_to_dev(kobj);
272	struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
273	struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
274	ssize_t ret;
275
276	xe_pm_runtime_get(pc_to_xe(pc));
277	ret = sysfs_emit(buff, "%s\n", gtidle->name);
278	xe_pm_runtime_put(pc_to_xe(pc));
279
280	return ret;
281}
282static struct kobj_attribute name_attr = __ATTR_RO(name);
283
284static ssize_t idle_status_show(struct kobject *kobj,
285				struct kobj_attribute *attr, char *buff)
286{
287	struct device *dev = kobj_to_dev(kobj);
288	struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
289	struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
290	enum xe_gt_idle_state state;
291
292	xe_pm_runtime_get(pc_to_xe(pc));
293	state = gtidle->idle_status(pc);
294	xe_pm_runtime_put(pc_to_xe(pc));
295
296	return sysfs_emit(buff, "%s\n", gt_idle_state_to_string(state));
297}
298static struct kobj_attribute idle_status_attr = __ATTR_RO(idle_status);
299
300u64 xe_gt_idle_residency_msec(struct xe_gt_idle *gtidle)
301{
302	struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
303	u64 residency;
304	unsigned long flags;
305
306	raw_spin_lock_irqsave(&gtidle->lock, flags);
307	residency = get_residency_ms(gtidle, gtidle->idle_residency(pc));
308	raw_spin_unlock_irqrestore(&gtidle->lock, flags);
309
310	return residency;
311}
312
313
314static ssize_t idle_residency_ms_show(struct kobject *kobj,
315				      struct kobj_attribute *attr, char *buff)
316{
317	struct device *dev = kobj_to_dev(kobj);
318	struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
319	struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
320	u64 residency;
321
322	xe_pm_runtime_get(pc_to_xe(pc));
323	residency = xe_gt_idle_residency_msec(gtidle);
324	xe_pm_runtime_put(pc_to_xe(pc));
325
326	return sysfs_emit(buff, "%llu\n", residency);
327}
328static struct kobj_attribute idle_residency_attr = __ATTR_RO(idle_residency_ms);
329
330static const struct attribute *gt_idle_attrs[] = {
331	&name_attr.attr,
332	&idle_status_attr.attr,
333	&idle_residency_attr.attr,
334	NULL,
335};
336
337static void gt_idle_fini(void *arg)
338{
339	struct kobject *kobj = arg;
340	struct xe_gt *gt = kobj_to_gt(kobj->parent);
341
342	xe_gt_idle_disable_pg(gt);
343
344	if (gt_to_xe(gt)->info.skip_guc_pc)
345		xe_gt_idle_disable_c6(gt);
346
347	sysfs_remove_files(kobj, gt_idle_attrs);
348	kobject_put(kobj);
349}
350
351int xe_gt_idle_init(struct xe_gt_idle *gtidle)
352{
353	struct xe_gt *gt = gtidle_to_gt(gtidle);
354	struct xe_device *xe = gt_to_xe(gt);
355	struct kobject *kobj;
356	int err;
357
358	if (IS_SRIOV_VF(xe))
359		return 0;
360
361	kobj = kobject_create_and_add("gtidle", gt->sysfs);
362	if (!kobj)
363		return -ENOMEM;
364
365	raw_spin_lock_init(&gtidle->lock);
366
367	if (xe_gt_is_media_type(gt)) {
368		snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-mc", gt->info.id);
369		gtidle->idle_residency = xe_guc_pc_mc6_residency;
370	} else {
371		snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-rc", gt->info.id);
372		gtidle->idle_residency = xe_guc_pc_rc6_residency;
373	}
374
375	/* Multiplier for Residency counter in units of 1.28us */
376	gtidle->residency_multiplier = 1280;
377	gtidle->idle_status = xe_guc_pc_c_status;
378
379	err = sysfs_create_files(kobj, gt_idle_attrs);
380	if (err) {
381		kobject_put(kobj);
382		return err;
383	}
384
385	xe_gt_idle_enable_pg(gt);
386
387	return devm_add_action_or_reset(xe->drm.dev, gt_idle_fini, kobj);
388}
389
390void xe_gt_idle_enable_c6(struct xe_gt *gt)
391{
392	xe_device_assert_mem_access(gt_to_xe(gt));
393	xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
394
395	if (IS_SRIOV_VF(gt_to_xe(gt)))
396		return;
397
398	/* Units of 1280 ns for a total of 5s */
399	xe_mmio_write32(&gt->mmio, RC_IDLE_HYSTERSIS, 0x3B9ACA);
400	/* Enable RC6 */
401	xe_mmio_write32(&gt->mmio, RC_CONTROL,
402			RC_CTL_HW_ENABLE | RC_CTL_TO_MODE | RC_CTL_RC6_ENABLE);
403}
404
405int xe_gt_idle_disable_c6(struct xe_gt *gt)
406{
407	unsigned int fw_ref;
408
409	xe_device_assert_mem_access(gt_to_xe(gt));
410
411	if (IS_SRIOV_VF(gt_to_xe(gt)))
412		return 0;
413
414	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
415	if (!fw_ref)
416		return -ETIMEDOUT;
417
418	xe_mmio_write32(&gt->mmio, RC_CONTROL, 0);
419	xe_mmio_write32(&gt->mmio, RC_STATE, 0);
420
421	xe_force_wake_put(gt_to_fw(gt), fw_ref);
422
423	return 0;
424}