xe_reg_sr.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2022 Intel Corporation
  4 */
  5
  6#include "xe_reg_sr.h"
  7
  8#include <kunit/visibility.h>
  9#include <linux/align.h>
 10#include <linux/string_helpers.h>
 11#include <linux/xarray.h>
 12
 13#include <drm/drm_managed.h>
 14#include <drm/drm_print.h>
 15
 16#include "regs/xe_engine_regs.h"
 17#include "regs/xe_gt_regs.h"
 18#include "xe_device.h"
 19#include "xe_device_types.h"
 20#include "xe_force_wake.h"
 21#include "xe_gt.h"
 22#include "xe_gt_mcr.h"
 23#include "xe_gt_printk.h"
 24#include "xe_hw_engine_types.h"
 25#include "xe_macros.h"
 26#include "xe_mmio.h"
 27#include "xe_rtp_types.h"
 28
 29static void reg_sr_fini(struct drm_device *drm, void *arg)
 30{
 31	struct xe_reg_sr *sr = arg;
 32	struct xe_reg_sr_entry *entry;
 33	unsigned long reg;
 34
 35	xa_for_each(&sr->xa, reg, entry)
 36		kfree(entry);
 37
 38	xa_destroy(&sr->xa);
 39}
 40
 41int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe)
 42{
 43	xa_init(&sr->xa);
 44	sr->name = name;
 45
 46	return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr);
 47}
 48EXPORT_SYMBOL_IF_KUNIT(xe_reg_sr_init);
 49
 50static bool compatible_entries(const struct xe_reg_sr_entry *e1,
 51			       const struct xe_reg_sr_entry *e2)
 52{
 53	/*
 54	 * Don't allow overwriting values: clr_bits/set_bits should be disjoint
 55	 * when operating in the same register
 56	 */
 57	if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits ||
 58	    e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits)
 59		return false;
 60
 61	if (e1->reg.raw != e2->reg.raw)
 62		return false;
 63
 64	return true;
 65}
 66
 67static void reg_sr_inc_error(struct xe_reg_sr *sr)
 68{
 69#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
 70	sr->errors++;
 71#endif
 72}
 73
 74int xe_reg_sr_add(struct xe_reg_sr *sr,
 75		  const struct xe_reg_sr_entry *e,
 76		  struct xe_gt *gt)
 77{
 78	unsigned long idx = e->reg.addr;
 79	struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx);
 80	int ret;
 81
 82	if (pentry) {
 83		if (!compatible_entries(pentry, e)) {
 84			ret = -EINVAL;
 85			goto fail;
 86		}
 87
 88		pentry->clr_bits |= e->clr_bits;
 89		pentry->set_bits |= e->set_bits;
 90		pentry->read_mask |= e->read_mask;
 91
 92		return 0;
 93	}
 94
 95	pentry = kmalloc(sizeof(*pentry), GFP_KERNEL);
 96	if (!pentry) {
 97		ret = -ENOMEM;
 98		goto fail;
 99	}
100
101	*pentry = *e;
102	ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL));
103	if (ret)
104		goto fail_free;
105
106	return 0;
107
108fail_free:
109	kfree(pentry);
110fail:
111	xe_gt_err(gt,
112		  "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n",
113		  idx, e->clr_bits, e->set_bits,
114		  str_yes_no(e->reg.masked),
115		  str_yes_no(e->reg.mcr),
116		  ret);
117	reg_sr_inc_error(sr);
118
119	return ret;
120}
121
122/*
123 * Convert back from encoded value to type-safe, only to be used when reg.mcr
124 * is true
125 */
126static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg)
127{
128	return (const struct xe_reg_mcr){.__reg.raw = reg.raw };
129}
130
131static void apply_one_mmio(struct xe_gt *gt, struct xe_reg_sr_entry *entry)
132{
133	struct xe_reg reg = entry->reg;
134	struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg);
135	u32 val;
136
137	/*
138	 * If this is a masked register, need to set the upper 16 bits.
139	 * Set them to clr_bits since that is always a superset of the bits
140	 * being modified.
141	 *
142	 * When it's not masked, we have to read it from hardware, unless we are
143	 * supposed to set all bits.
144	 */
145	if (reg.masked)
146		val = entry->clr_bits << 16;
147	else if (entry->clr_bits + 1)
148		val = (reg.mcr ?
149		       xe_gt_mcr_unicast_read_any(gt, reg_mcr) :
150		       xe_mmio_read32(&gt->mmio, reg)) & (~entry->clr_bits);
151	else
152		val = 0;
153
154	/*
155	 * TODO: add selftest to validate all tables, regardless of platform:
156	 *   - Masked registers can't have set_bits with upper bits set
157	 *   - set_bits must be contained in clr_bits
158	 */
159	val |= entry->set_bits;
160
161	xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val);
162
163	if (entry->reg.mcr)
164		xe_gt_mcr_multicast_write(gt, reg_mcr, val);
165	else
166		xe_mmio_write32(&gt->mmio, reg, val);
167}
168
169void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt)
170{
171	struct xe_reg_sr_entry *entry;
172	unsigned long reg;
173	unsigned int fw_ref;
174
175	if (xa_empty(&sr->xa))
176		return;
177
178	if (IS_SRIOV_VF(gt_to_xe(gt)))
179		return;
180
181	xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name);
182
183	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
184	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
185		goto err_force_wake;
186
187	xa_for_each(&sr->xa, reg, entry)
188		apply_one_mmio(gt, entry);
189
190	xe_force_wake_put(gt_to_fw(gt), fw_ref);
191
192	return;
193
194err_force_wake:
195	xe_force_wake_put(gt_to_fw(gt), fw_ref);
196	xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n");
197}
198
199/**
200 * xe_reg_sr_dump - print all save/restore entries
201 * @sr: Save/restore entries
202 * @p: DRM printer
203 */
204void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p)
205{
206	struct xe_reg_sr_entry *entry;
207	unsigned long reg;
208
209	if (!sr->name || xa_empty(&sr->xa))
210		return;
211
212	drm_printf(p, "%s\n", sr->name);
213	xa_for_each(&sr->xa, reg, entry)
214		drm_printf(p, "\tREG[0x%lx] clr=0x%08x set=0x%08x masked=%s mcr=%s\n",
215			   reg, entry->clr_bits, entry->set_bits,
216			   str_yes_no(entry->reg.masked),
217			   str_yes_no(entry->reg.mcr));
218}