umc.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * AMD Address Translation Library
  4 *
  5 * umc.c : Unified Memory Controller (UMC) topology helpers
  6 *
  7 * Copyright (c) 2023, Advanced Micro Devices, Inc.
  8 * All Rights Reserved.
  9 *
 10 * Author: Yazen Ghannam <Yazen.Ghannam@amd.com>
 11 */
 12
 13#include "internal.h"
 14
 15/*
 16 * MI300 has a fixed, model-specific mapping between a UMC instance and
 17 * its related Data Fabric Coherent Station instance.
 18 *
 19 * The MCA_IPID_UMC[InstanceId] field holds a unique identifier for the
 20 * UMC instance within a Node. Use this to find the appropriate Coherent
 21 * Station ID.
 22 *
 23 * Redundant bits were removed from the map below.
 24 */
 25static const u16 umc_coh_st_map[32] = {
 26	0x393, 0x293, 0x193, 0x093,
 27	0x392, 0x292, 0x192, 0x092,
 28	0x391, 0x291, 0x191, 0x091,
 29	0x390, 0x290, 0x190, 0x090,
 30	0x793, 0x693, 0x593, 0x493,
 31	0x792, 0x692, 0x592, 0x492,
 32	0x791, 0x691, 0x591, 0x491,
 33	0x790, 0x690, 0x590, 0x490,
 34};
 35
 36#define UMC_ID_MI300 GENMASK(23, 12)
 37static u8 get_coh_st_inst_id_mi300(struct atl_err *err)
 38{
 39	u16 umc_id = FIELD_GET(UMC_ID_MI300, err->ipid);
 40	u8 i;
 41
 42	for (i = 0; i < ARRAY_SIZE(umc_coh_st_map); i++) {
 43		if (umc_id == umc_coh_st_map[i])
 44			break;
 45	}
 46
 47	WARN_ON_ONCE(i >= ARRAY_SIZE(umc_coh_st_map));
 48
 49	return i;
 50}
 51
 52
 53struct xor_bits {
 54	bool	xor_enable;
 55	u16	col_xor;
 56	u32	row_xor;
 57};
 58
 59#define NUM_BANK_BITS	4
 60#define NUM_COL_BITS	5
 61#define NUM_SID_BITS	2
 62
 63static struct {
 64	/* UMC::CH::AddrHashBank */
 65	struct xor_bits	bank[NUM_BANK_BITS];
 66
 67	/* UMC::CH::AddrHashPC */
 68	struct xor_bits	pc;
 69
 70	/* UMC::CH::AddrHashPC2 */
 71	u8		bank_xor;
 72} addr_hash;
 73
 74static struct {
 75	u8 bank[NUM_BANK_BITS];
 76	u8 col[NUM_COL_BITS];
 77	u8 sid[NUM_SID_BITS];
 78	u8 num_row_lo;
 79	u8 num_row_hi;
 80	u8 row_lo;
 81	u8 row_hi;
 82	u8 pc;
 83} bit_shifts;
 84
 85#define MI300_UMC_CH_BASE	0x90000
 86#define MI300_ADDR_CFG		(MI300_UMC_CH_BASE + 0x30)
 87#define MI300_ADDR_SEL		(MI300_UMC_CH_BASE + 0x40)
 88#define MI300_COL_SEL_LO	(MI300_UMC_CH_BASE + 0x50)
 89#define MI300_ADDR_SEL_2	(MI300_UMC_CH_BASE + 0xA4)
 90#define MI300_ADDR_HASH_BANK0	(MI300_UMC_CH_BASE + 0xC8)
 91#define MI300_ADDR_HASH_PC	(MI300_UMC_CH_BASE + 0xE0)
 92#define MI300_ADDR_HASH_PC2	(MI300_UMC_CH_BASE + 0xE4)
 93
 94#define ADDR_HASH_XOR_EN	BIT(0)
 95#define ADDR_HASH_COL_XOR	GENMASK(13, 1)
 96#define ADDR_HASH_ROW_XOR	GENMASK(31, 14)
 97#define ADDR_HASH_BANK_XOR	GENMASK(5, 0)
 98
 99#define ADDR_CFG_NUM_ROW_LO	GENMASK(11, 8)
100#define ADDR_CFG_NUM_ROW_HI	GENMASK(15, 12)
101
102#define ADDR_SEL_BANK0		GENMASK(3, 0)
103#define ADDR_SEL_BANK1		GENMASK(7, 4)
104#define ADDR_SEL_BANK2		GENMASK(11, 8)
105#define ADDR_SEL_BANK3		GENMASK(15, 12)
106#define ADDR_SEL_BANK4		GENMASK(20, 16)
107#define ADDR_SEL_ROW_LO		GENMASK(27, 24)
108#define ADDR_SEL_ROW_HI		GENMASK(31, 28)
109
110#define COL_SEL_LO_COL0		GENMASK(3, 0)
111#define COL_SEL_LO_COL1		GENMASK(7, 4)
112#define COL_SEL_LO_COL2		GENMASK(11, 8)
113#define COL_SEL_LO_COL3		GENMASK(15, 12)
114#define COL_SEL_LO_COL4		GENMASK(19, 16)
115
116#define ADDR_SEL_2_BANK5	GENMASK(4, 0)
117#define ADDR_SEL_2_CHAN		GENMASK(15, 12)
118
119/*
120 * Read UMC::CH::AddrHash{Bank,PC,PC2} registers to get XOR bits used
121 * for hashing.
122 *
123 * Also, read UMC::CH::Addr{Cfg,Sel,Sel2} and UMC::CH:ColSelLo registers to
124 * get the values needed to reconstruct the normalized address. Apply additional
125 * offsets to the raw register values, as needed.
126 *
127 * Do this during module init, since the values will not change during run time.
128 *
129 * These registers are instantiated for each UMC across each AMD Node.
130 * However, they should be identically programmed due to the fixed hardware
131 * design of MI300 systems. So read the values from Node 0 UMC 0 and keep a
132 * single global structure for simplicity.
133 */
134int get_umc_info_mi300(void)
135{
136	u32 temp;
137	int ret;
138	u8 i;
139
140	for (i = 0; i < NUM_BANK_BITS; i++) {
141		ret = amd_smn_read(0, MI300_ADDR_HASH_BANK0 + (i * 4), &temp);
142		if (ret)
143			return ret;
144
145		addr_hash.bank[i].xor_enable = FIELD_GET(ADDR_HASH_XOR_EN,  temp);
146		addr_hash.bank[i].col_xor    = FIELD_GET(ADDR_HASH_COL_XOR, temp);
147		addr_hash.bank[i].row_xor    = FIELD_GET(ADDR_HASH_ROW_XOR, temp);
148	}
149
150	ret = amd_smn_read(0, MI300_ADDR_HASH_PC, &temp);
151	if (ret)
152		return ret;
153
154	addr_hash.pc.xor_enable = FIELD_GET(ADDR_HASH_XOR_EN,  temp);
155	addr_hash.pc.col_xor    = FIELD_GET(ADDR_HASH_COL_XOR, temp);
156	addr_hash.pc.row_xor    = FIELD_GET(ADDR_HASH_ROW_XOR, temp);
157
158	ret = amd_smn_read(0, MI300_ADDR_HASH_PC2, &temp);
159	if (ret)
160		return ret;
161
162	addr_hash.bank_xor = FIELD_GET(ADDR_HASH_BANK_XOR, temp);
163
164	ret = amd_smn_read(0, MI300_ADDR_CFG, &temp);
165	if (ret)
166		return ret;
167
168	bit_shifts.num_row_hi = FIELD_GET(ADDR_CFG_NUM_ROW_HI, temp);
169	bit_shifts.num_row_lo = 10 + FIELD_GET(ADDR_CFG_NUM_ROW_LO, temp);
170
171	ret = amd_smn_read(0, MI300_ADDR_SEL, &temp);
172	if (ret)
173		return ret;
174
175	bit_shifts.bank[0] = 5 + FIELD_GET(ADDR_SEL_BANK0, temp);
176	bit_shifts.bank[1] = 5 + FIELD_GET(ADDR_SEL_BANK1, temp);
177	bit_shifts.bank[2] = 5 + FIELD_GET(ADDR_SEL_BANK2, temp);
178	bit_shifts.bank[3] = 5 + FIELD_GET(ADDR_SEL_BANK3, temp);
179	/* Use BankBit4 for the SID0 position. */
180	bit_shifts.sid[0]  = 5 + FIELD_GET(ADDR_SEL_BANK4, temp);
181	bit_shifts.row_lo  = 12 + FIELD_GET(ADDR_SEL_ROW_LO, temp);
182	bit_shifts.row_hi  = 24 + FIELD_GET(ADDR_SEL_ROW_HI, temp);
183
184	ret = amd_smn_read(0, MI300_COL_SEL_LO, &temp);
185	if (ret)
186		return ret;
187
188	bit_shifts.col[0] = 2 + FIELD_GET(COL_SEL_LO_COL0, temp);
189	bit_shifts.col[1] = 2 + FIELD_GET(COL_SEL_LO_COL1, temp);
190	bit_shifts.col[2] = 2 + FIELD_GET(COL_SEL_LO_COL2, temp);
191	bit_shifts.col[3] = 2 + FIELD_GET(COL_SEL_LO_COL3, temp);
192	bit_shifts.col[4] = 2 + FIELD_GET(COL_SEL_LO_COL4, temp);
193
194	ret = amd_smn_read(0, MI300_ADDR_SEL_2, &temp);
195	if (ret)
196		return ret;
197
198	/* Use BankBit5 for the SID1 position. */
199	bit_shifts.sid[1] = 5 + FIELD_GET(ADDR_SEL_2_BANK5, temp);
200	bit_shifts.pc	  = 5 + FIELD_GET(ADDR_SEL_2_CHAN, temp);
201
202	return 0;
203}
204
205/*
206 * MI300 systems report a DRAM address in MCA_ADDR for DRAM ECC errors. This must
207 * be converted to the intermediate normalized address (NA) before translating to a
208 * system physical address.
209 *
210 * The DRAM address includes bank, row, and column. Also included are bits for
211 * pseudochannel (PC) and stack ID (SID).
212 *
213 * Abbreviations: (S)tack ID, (P)seudochannel, (R)ow, (B)ank, (C)olumn, (Z)ero
214 *
215 * The MCA address format is as follows:
216 *	MCA_ADDR[27:0] = {S[1:0], P[0], R[14:0], B[3:0], C[4:0], Z[0]}
217 *
218 * Additionally, the PC and Bank bits may be hashed. This must be accounted for before
219 * reconstructing the normalized address.
220 */
221#define MI300_UMC_MCA_BANK	GENMASK(9, 6)
222#define MI300_UMC_MCA_ROW	GENMASK(24, 10)
223#define MI300_UMC_MCA_PC	BIT(25)
224#define MI300_UMC_MCA_SID	GENMASK(27, 26)
225
226static unsigned long convert_dram_to_norm_addr_mi300(unsigned long addr)
227{
228	u16 i, col, row, bank, pc, sid;
229	u32 temp;
230
231	col  = FIELD_GET(MI300_UMC_MCA_COL,  addr);
232	bank = FIELD_GET(MI300_UMC_MCA_BANK, addr);
233	row  = FIELD_GET(MI300_UMC_MCA_ROW,  addr);
234	pc   = FIELD_GET(MI300_UMC_MCA_PC,   addr);
235	sid  = FIELD_GET(MI300_UMC_MCA_SID,  addr);
236
237	/* Calculate hash for each Bank bit. */
238	for (i = 0; i < NUM_BANK_BITS; i++) {
239		if (!addr_hash.bank[i].xor_enable)
240			continue;
241
242		temp  = hweight16(col & addr_hash.bank[i].col_xor) & 1;
243		temp ^= hweight16(row & addr_hash.bank[i].row_xor) & 1;
244		bank ^= temp << i;
245	}
246
247	/* Calculate hash for PC bit. */
248	if (addr_hash.pc.xor_enable) {
249		temp  = hweight16(col & addr_hash.pc.col_xor) & 1;
250		temp ^= hweight16(row & addr_hash.pc.row_xor) & 1;
251		/* Bits SID[1:0] act as Bank[5:4] for PC hash, so apply them here. */
252		temp ^= hweight16((bank | sid << NUM_BANK_BITS) & addr_hash.bank_xor) & 1;
253		pc   ^= temp;
254	}
255
256	/* Reconstruct the normalized address starting with NA[4:0] = 0 */
257	addr  = 0;
258
259	/* Column bits */
260	for (i = 0; i < NUM_COL_BITS; i++) {
261		temp  = (col >> i) & 0x1;
262		addr |= temp << bit_shifts.col[i];
263	}
264
265	/* Bank bits */
266	for (i = 0; i < NUM_BANK_BITS; i++) {
267		temp  = (bank >> i) & 0x1;
268		addr |= temp << bit_shifts.bank[i];
269	}
270
271	/* Row lo bits */
272	for (i = 0; i < bit_shifts.num_row_lo; i++) {
273		temp  = (row >> i) & 0x1;
274		addr |= temp << (i + bit_shifts.row_lo);
275	}
276
277	/* Row hi bits */
278	for (i = 0; i < bit_shifts.num_row_hi; i++) {
279		temp  = (row >> (i + bit_shifts.num_row_lo)) & 0x1;
280		addr |= temp << (i + bit_shifts.row_hi);
281	}
282
283	/* PC bit */
284	addr |= pc << bit_shifts.pc;
285
286	/* SID bits */
287	for (i = 0; i < NUM_SID_BITS; i++) {
288		temp  = (sid >> i) & 0x1;
289		addr |= temp << bit_shifts.sid[i];
290	}
291
292	pr_debug("Addr=0x%016lx", addr);
293	pr_debug("Bank=%u Row=%u Column=%u PC=%u SID=%u", bank, row, col, pc, sid);
294
295	return addr;
296}
297
298/*
299 * When a DRAM ECC error occurs on MI300 systems, it is recommended to retire
300 * all memory within that DRAM row. This applies to the memory with a DRAM
301 * bank.
302 *
303 * To find the memory addresses, loop through permutations of the DRAM column
304 * bits and find the System Physical address of each. The column bits are used
305 * to calculate the intermediate Normalized address, so all permutations should
306 * be checked.
307 *
308 * See amd_atl::convert_dram_to_norm_addr_mi300() for MI300 address formats.
309 */
310#define MI300_NUM_COL		BIT(HWEIGHT(MI300_UMC_MCA_COL))
311static void _retire_row_mi300(struct atl_err *a_err)
312{
313	unsigned long addr;
314	struct page *p;
315	u8 col;
316
317	for (col = 0; col < MI300_NUM_COL; col++) {
318		a_err->addr &= ~MI300_UMC_MCA_COL;
319		a_err->addr |= FIELD_PREP(MI300_UMC_MCA_COL, col);
320
321		addr = amd_convert_umc_mca_addr_to_sys_addr(a_err);
322		if (IS_ERR_VALUE(addr))
323			continue;
324
325		addr = PHYS_PFN(addr);
326
327		/*
328		 * Skip invalid or already poisoned pages to avoid unnecessary
329		 * error messages from memory_failure().
330		 */
331		p = pfn_to_online_page(addr);
332		if (!p)
333			continue;
334
335		if (PageHWPoison(p))
336			continue;
337
338		memory_failure(addr, 0);
339	}
340}
341
342/*
343 * In addition to the column bits, the row[13] bit should also be included when
344 * calculating addresses affected by a physical row.
345 *
346 * Instead of running through another loop over a single bit, just run through
347 * the column bits twice and flip the row[13] bit in-between.
348 *
349 * See MI300_UMC_MCA_ROW for the row bits in MCA_ADDR_UMC value.
350 */
351static void retire_row_mi300(struct atl_err *a_err)
352{
353	_retire_row_mi300(a_err);
354	a_err->addr ^= MI300_UMC_MCA_ROW13;
355	_retire_row_mi300(a_err);
356}
357
358void amd_retire_dram_row(struct atl_err *a_err)
359{
360	if (df_cfg.rev == DF4p5 && df_cfg.flags.heterogeneous)
361		return retire_row_mi300(a_err);
362}
363EXPORT_SYMBOL_GPL(amd_retire_dram_row);
364
365static unsigned long get_addr(unsigned long addr)
366{
367	if (df_cfg.rev == DF4p5 && df_cfg.flags.heterogeneous)
368		return convert_dram_to_norm_addr_mi300(addr);
369
370	return addr;
371}
372
373#define MCA_IPID_INST_ID_HI	GENMASK_ULL(47, 44)
374static u8 get_die_id(struct atl_err *err)
375{
376	/*
377	 * AMD Node ID is provided in MCA_IPID[InstanceIdHi], and this
378	 * needs to be divided by 4 to get the internal Die ID.
379	 */
380	if (df_cfg.rev == DF4p5 && df_cfg.flags.heterogeneous) {
381		u8 node_id = FIELD_GET(MCA_IPID_INST_ID_HI, err->ipid);
382
383		return node_id >> 2;
384	}
385
386	/*
387	 * For CPUs, this is the AMD Node ID modulo the number
388	 * of AMD Nodes per socket.
389	 */
390	return topology_amd_node_id(err->cpu) % topology_amd_nodes_per_pkg();
391}
392
393#define UMC_CHANNEL_NUM	GENMASK(31, 20)
394static u8 get_coh_st_inst_id(struct atl_err *err)
395{
396	if (df_cfg.rev == DF4p5 && df_cfg.flags.heterogeneous)
397		return get_coh_st_inst_id_mi300(err);
398
399	return FIELD_GET(UMC_CHANNEL_NUM, err->ipid);
400}
401
402unsigned long convert_umc_mca_addr_to_sys_addr(struct atl_err *err)
403{
404	u8 socket_id = topology_physical_package_id(err->cpu);
405	u8 coh_st_inst_id = get_coh_st_inst_id(err);
406	unsigned long addr = get_addr(err->addr);
407	u8 die_id = get_die_id(err);
408	unsigned long ret_addr;
409
410	pr_debug("socket_id=0x%x die_id=0x%x coh_st_inst_id=0x%x addr=0x%016lx",
411		 socket_id, die_id, coh_st_inst_id, addr);
412
413	ret_addr = prm_umc_norm_to_sys_addr(socket_id, err->ipid, addr);
414	if (!IS_ERR_VALUE(ret_addr) || df_cfg.flags.prm_only)
415		return ret_addr;
416
417	return norm_to_sys_addr(socket_id, die_id, coh_st_inst_id, addr);
418}