adf_bank_state.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright(c) 2025 Intel Corporation */
  3
  4#define pr_fmt(fmt)	"QAT: " fmt
  5
  6#include <linux/bits.h>
  7#include <linux/dev_printk.h>
  8#include <linux/printk.h>
  9#include "adf_accel_devices.h"
 10#include "adf_bank_state.h"
 11#include "adf_common_drv.h"
 12
 13/* Ring interrupt masks */
 14#define ADF_RP_INT_SRC_SEL_F_RISE_MASK	GENMASK(1, 0)
 15#define ADF_RP_INT_SRC_SEL_F_FALL_MASK	GENMASK(2, 0)
 16#define ADF_RP_INT_SRC_SEL_RANGE_WIDTH	4
 17
 18static inline int check_stat(u32 (*op)(void __iomem *, u32), u32 expect_val,
 19			     const char *name, void __iomem *base, u32 bank)
 20{
 21	u32 actual_val = op(base, bank);
 22
 23	if (expect_val == actual_val)
 24		return 0;
 25
 26	pr_err("Fail to restore %s register. Expected %#x, actual %#x\n",
 27	       name, expect_val, actual_val);
 28
 29	return -EINVAL;
 30}
 31
 32static void bank_state_save(struct adf_hw_csr_ops *ops, void __iomem *base,
 33			    u32 bank, struct adf_bank_state *state, u32 num_rings)
 34{
 35	u32 i;
 36
 37	state->ringstat0 = ops->read_csr_stat(base, bank);
 38	state->ringuostat = ops->read_csr_uo_stat(base, bank);
 39	state->ringestat = ops->read_csr_e_stat(base, bank);
 40	state->ringnestat = ops->read_csr_ne_stat(base, bank);
 41	state->ringnfstat = ops->read_csr_nf_stat(base, bank);
 42	state->ringfstat = ops->read_csr_f_stat(base, bank);
 43	state->ringcstat0 = ops->read_csr_c_stat(base, bank);
 44	state->iaintflagen = ops->read_csr_int_en(base, bank);
 45	state->iaintflagreg = ops->read_csr_int_flag(base, bank);
 46	state->iaintflagsrcsel0 = ops->read_csr_int_srcsel(base, bank);
 47	state->iaintcolen = ops->read_csr_int_col_en(base, bank);
 48	state->iaintcolctl = ops->read_csr_int_col_ctl(base, bank);
 49	state->iaintflagandcolen = ops->read_csr_int_flag_and_col(base, bank);
 50	state->ringexpstat = ops->read_csr_exp_stat(base, bank);
 51	state->ringexpintenable = ops->read_csr_exp_int_en(base, bank);
 52	state->ringsrvarben = ops->read_csr_ring_srv_arb_en(base, bank);
 53
 54	for (i = 0; i < num_rings; i++) {
 55		state->rings[i].head = ops->read_csr_ring_head(base, bank, i);
 56		state->rings[i].tail = ops->read_csr_ring_tail(base, bank, i);
 57		state->rings[i].config = ops->read_csr_ring_config(base, bank, i);
 58		state->rings[i].base = ops->read_csr_ring_base(base, bank, i);
 59	}
 60}
 61
 62static int bank_state_restore(struct adf_hw_csr_ops *ops, void __iomem *base,
 63			      u32 bank, struct adf_bank_state *state, u32 num_rings,
 64			      int tx_rx_gap)
 65{
 66	u32 val, tmp_val, i;
 67	int ret;
 68
 69	for (i = 0; i < num_rings; i++)
 70		ops->write_csr_ring_base(base, bank, i, state->rings[i].base);
 71
 72	for (i = 0; i < num_rings; i++)
 73		ops->write_csr_ring_config(base, bank, i, state->rings[i].config);
 74
 75	for (i = 0; i < num_rings / 2; i++) {
 76		int tx = i * (tx_rx_gap + 1);
 77		int rx = tx + tx_rx_gap;
 78
 79		ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head);
 80		ops->write_csr_ring_tail(base, bank, tx, state->rings[tx].tail);
 81
 82		/*
 83		 * The TX ring head needs to be updated again to make sure that
 84		 * the HW will not consider the ring as full when it is empty
 85		 * and the correct state flags are set to match the recovered state.
 86		 */
 87		if (state->ringestat & BIT(tx)) {
 88			val = ops->read_csr_int_srcsel(base, bank);
 89			val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK;
 90			ops->write_csr_int_srcsel_w_val(base, bank, val);
 91			ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head);
 92		}
 93
 94		ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail);
 95		val = ops->read_csr_int_srcsel(base, bank);
 96		val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH;
 97		ops->write_csr_int_srcsel_w_val(base, bank, val);
 98
 99		ops->write_csr_ring_head(base, bank, rx, state->rings[rx].head);
100		val = ops->read_csr_int_srcsel(base, bank);
101		val |= ADF_RP_INT_SRC_SEL_F_FALL_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH;
102		ops->write_csr_int_srcsel_w_val(base, bank, val);
103
104		/*
105		 * The RX ring tail needs to be updated again to make sure that
106		 * the HW will not consider the ring as empty when it is full
107		 * and the correct state flags are set to match the recovered state.
108		 */
109		if (state->ringfstat & BIT(rx))
110			ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail);
111	}
112
113	ops->write_csr_int_flag_and_col(base, bank, state->iaintflagandcolen);
114	ops->write_csr_int_en(base, bank, state->iaintflagen);
115	ops->write_csr_int_col_en(base, bank, state->iaintcolen);
116	ops->write_csr_int_srcsel_w_val(base, bank, state->iaintflagsrcsel0);
117	ops->write_csr_exp_int_en(base, bank, state->ringexpintenable);
118	ops->write_csr_int_col_ctl(base, bank, state->iaintcolctl);
119
120	/*
121	 * Verify whether any exceptions were raised during the bank save process.
122	 * If exceptions occurred, the status and exception registers cannot
123	 * be directly restored. Consequently, further restoration is not
124	 * feasible, and the current state of the ring should be maintained.
125	 */
126	val = state->ringexpstat;
127	if (val) {
128		pr_info("Bank %u state not fully restored due to exception in saved state (%#x)\n",
129			bank, val);
130		return 0;
131	}
132
133	/* Ensure that the restoration process completed without exceptions */
134	tmp_val = ops->read_csr_exp_stat(base, bank);
135	if (tmp_val) {
136		pr_err("Bank %u restored with exception: %#x\n", bank, tmp_val);
137		return -EFAULT;
138	}
139
140	ops->write_csr_ring_srv_arb_en(base, bank, state->ringsrvarben);
141
142	/* Check that all ring statuses match the saved state. */
143	ret = check_stat(ops->read_csr_stat, state->ringstat0, "ringstat",
144			 base, bank);
145	if (ret)
146		return ret;
147
148	ret = check_stat(ops->read_csr_e_stat, state->ringestat, "ringestat",
149			 base, bank);
150	if (ret)
151		return ret;
152
153	ret = check_stat(ops->read_csr_ne_stat, state->ringnestat, "ringnestat",
154			 base, bank);
155	if (ret)
156		return ret;
157
158	ret = check_stat(ops->read_csr_nf_stat, state->ringnfstat, "ringnfstat",
159			 base, bank);
160	if (ret)
161		return ret;
162
163	ret = check_stat(ops->read_csr_f_stat, state->ringfstat, "ringfstat",
164			 base, bank);
165	if (ret)
166		return ret;
167
168	ret = check_stat(ops->read_csr_c_stat, state->ringcstat0, "ringcstat",
169			 base, bank);
170	if (ret)
171		return ret;
172
173	return 0;
174}
175
176/**
177 * adf_bank_state_save() - save state of bank-related registers
178 * @accel_dev: Pointer to the device structure
179 * @bank_number: Bank number
180 * @state: Pointer to bank state structure
181 *
182 * This function saves the state of a bank by reading the bank CSRs and
183 * writing them in the @state structure.
184 *
185 * Returns 0 on success, error code otherwise
186 */
187int adf_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number,
188			struct adf_bank_state *state)
189{
190	struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
191	struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
192	void __iomem *csr_base = adf_get_etr_base(accel_dev);
193
194	if (bank_number >= hw_data->num_banks || !state)
195		return -EINVAL;
196
197	dev_dbg(&GET_DEV(accel_dev), "Saving state of bank %d\n", bank_number);
198
199	bank_state_save(csr_ops, csr_base, bank_number, state,
200			hw_data->num_rings_per_bank);
201
202	return 0;
203}
204EXPORT_SYMBOL_GPL(adf_bank_state_save);
205
206/**
207 * adf_bank_state_restore() - restore state of bank-related registers
208 * @accel_dev: Pointer to the device structure
209 * @bank_number: Bank number
210 * @state: Pointer to bank state structure
211 *
212 * This function attempts to restore the state of a bank by writing the
213 * bank CSRs to the values in the state structure.
214 *
215 * Returns 0 on success, error code otherwise
216 */
217int adf_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number,
218			   struct adf_bank_state *state)
219{
220	struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
221	struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
222	void __iomem *csr_base = adf_get_etr_base(accel_dev);
223	int ret;
224
225	if (bank_number >= hw_data->num_banks  || !state)
226		return -EINVAL;
227
228	dev_dbg(&GET_DEV(accel_dev), "Restoring state of bank %d\n", bank_number);
229
230	ret = bank_state_restore(csr_ops, csr_base, bank_number, state,
231				 hw_data->num_rings_per_bank, hw_data->tx_rx_gap);
232	if (ret)
233		dev_err(&GET_DEV(accel_dev),
234			"Unable to restore state of bank %d\n", bank_number);
235
236	return ret;
237}
238EXPORT_SYMBOL_GPL(adf_bank_state_restore);