amdgpu_umc.c

  1/*
  2 * Copyright 2019 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 */
 23
 24#include <linux/sort.h>
 25#include "amdgpu.h"
 26#include "umc_v6_7.h"
 27#include "amdgpu_ras_mgr.h"
 28#define MAX_UMC_POISON_POLLING_TIME_SYNC   20  //ms
 29
 30#define MAX_UMC_HASH_STRING_SIZE  256
 31
 32static int amdgpu_umc_convert_error_address(struct amdgpu_device *adev,
 33				    struct ras_err_data *err_data, uint64_t err_addr,
 34				    uint32_t ch_inst, uint32_t umc_inst)
 35{
 36	switch (amdgpu_ip_version(adev, UMC_HWIP, 0)) {
 37	case IP_VERSION(6, 7, 0):
 38		umc_v6_7_convert_error_address(adev,
 39				err_data, err_addr, ch_inst, umc_inst);
 40		break;
 41	default:
 42		dev_warn(adev->dev,
 43			 "UMC address to Physical address translation is not supported\n");
 44		return AMDGPU_RAS_FAIL;
 45	}
 46
 47	return AMDGPU_RAS_SUCCESS;
 48}
 49
 50int amdgpu_umc_page_retirement_mca(struct amdgpu_device *adev,
 51			uint64_t err_addr, uint32_t ch_inst, uint32_t umc_inst)
 52{
 53	struct ras_err_data err_data;
 54	int ret;
 55
 56	ret = amdgpu_ras_error_data_init(&err_data);
 57	if (ret)
 58		return ret;
 59
 60	err_data.err_addr =
 61		kcalloc(adev->umc.max_ras_err_cnt_per_query,
 62			sizeof(struct eeprom_table_record), GFP_KERNEL);
 63	if (!err_data.err_addr) {
 64		dev_warn(adev->dev,
 65			"Failed to alloc memory for umc error record in MCA notifier!\n");
 66		ret = AMDGPU_RAS_FAIL;
 67		goto out_fini_err_data;
 68	}
 69
 70	err_data.err_addr_len = adev->umc.max_ras_err_cnt_per_query;
 71
 72	/*
 73	 * Translate UMC channel address to Physical address
 74	 */
 75	ret = amdgpu_umc_convert_error_address(adev, &err_data, err_addr,
 76					ch_inst, umc_inst);
 77	if (ret)
 78		goto out_free_err_addr;
 79
 80	if (amdgpu_bad_page_threshold != 0) {
 81		amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
 82						err_data.err_addr_cnt, false);
 83		amdgpu_ras_save_bad_pages(adev, NULL);
 84	}
 85
 86out_free_err_addr:
 87	kfree(err_data.err_addr);
 88
 89out_fini_err_data:
 90	amdgpu_ras_error_data_fini(&err_data);
 91
 92	return ret;
 93}
 94
 95void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
 96			void *ras_error_status)
 97{
 98	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
 99	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
100	struct amdgpu_ras_eeprom_control *control = &con->eeprom_control;
101	unsigned int error_query_mode;
102	int ret = 0;
103	unsigned long err_count;
104
105	amdgpu_ras_get_error_query_mode(adev, &error_query_mode);
106
107	err_data->err_addr =
108		kcalloc(adev->umc.max_ras_err_cnt_per_query,
109			sizeof(struct eeprom_table_record), GFP_KERNEL);
110
111	/* still call query_ras_error_address to clear error status
112	 * even NOMEM error is encountered
113	 */
114	if (!err_data->err_addr)
115		dev_warn(adev->dev,
116			"Failed to alloc memory for umc error address record!\n");
117	else
118		err_data->err_addr_len = adev->umc.max_ras_err_cnt_per_query;
119
120	mutex_lock(&con->page_retirement_lock);
121	if (!amdgpu_ras_smu_eeprom_supported(adev)) {
122		ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(con->umc_ecc));
123		if (ret == -EOPNOTSUPP &&
124		    error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) {
125			if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
126			    adev->umc.ras->ras_block.hw_ops->query_ras_error_count)
127				adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev,
128								ras_error_status);
129
130			if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
131			    adev->umc.ras->ras_block.hw_ops->query_ras_error_address &&
132			    adev->umc.max_ras_err_cnt_per_query) {
133				err_data->err_addr =
134					kcalloc(adev->umc.max_ras_err_cnt_per_query,
135						sizeof(struct eeprom_table_record), GFP_KERNEL);
136
137				/* still call query_ras_error_address to clear error status
138				 * even NOMEM error is encountered
139				 */
140				if (!err_data->err_addr)
141					dev_warn(adev->dev,
142						"Failed to alloc memory for umc error address record!\n");
143				else
144					err_data->err_addr_len =
145						adev->umc.max_ras_err_cnt_per_query;
146
147				/* umc query_ras_error_address is also responsible for clearing
148				 * error status
149				 */
150				adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev,
151								ras_error_status);
152			}
153		} else if (error_query_mode == AMDGPU_RAS_FIRMWARE_ERROR_QUERY ||
154		    (!ret && error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY)) {
155			if (adev->umc.ras &&
156			    adev->umc.ras->ecc_info_query_ras_error_count)
157				adev->umc.ras->ecc_info_query_ras_error_count(adev,
158								ras_error_status);
159
160			if (adev->umc.ras &&
161			    adev->umc.ras->ecc_info_query_ras_error_address &&
162			    adev->umc.max_ras_err_cnt_per_query) {
163				err_data->err_addr =
164					kcalloc(adev->umc.max_ras_err_cnt_per_query,
165						sizeof(struct eeprom_table_record), GFP_KERNEL);
166
167				/* still call query_ras_error_address to clear error status
168				 * even NOMEM error is encountered
169				 */
170				if (!err_data->err_addr)
171					dev_warn(adev->dev,
172						"Failed to alloc memory for umc error address record!\n");
173				else
174					err_data->err_addr_len =
175						adev->umc.max_ras_err_cnt_per_query;
176
177				/* umc query_ras_error_address is also responsible for clearing
178				 * error status
179				 */
180				adev->umc.ras->ecc_info_query_ras_error_address(adev,
181								ras_error_status);
182			}
183		}
184	} else {
185		if (!amdgpu_ras_eeprom_update_record_num(control)) {
186			err_data->err_addr_cnt = err_data->de_count =
187				control->ras_num_recs -	control->ras_num_recs_old;
188			amdgpu_ras_eeprom_read_idx(control, err_data->err_addr,
189				control->ras_num_recs_old, err_data->de_count);
190		}
191	}
192
193	/* only uncorrectable error needs gpu reset */
194	if (err_data->ue_count || err_data->de_count) {
195		err_count = err_data->ue_count + err_data->de_count;
196		if ((amdgpu_bad_page_threshold != 0) &&
197			err_data->err_addr_cnt) {
198			amdgpu_ras_add_bad_pages(adev, err_data->err_addr,
199				err_data->err_addr_cnt, amdgpu_ras_smu_eeprom_supported(adev));
200			amdgpu_ras_save_bad_pages(adev, &err_count);
201
202			amdgpu_dpm_send_hbm_bad_pages_num(adev,
203					con->eeprom_control.ras_num_bad_pages);
204
205			if (con->update_channel_flag == true) {
206				amdgpu_dpm_send_hbm_bad_channel_flag(adev, con->eeprom_control.bad_channel_bitmap);
207				con->update_channel_flag = false;
208			}
209		}
210	}
211
212	kfree(err_data->err_addr);
213	err_data->err_addr = NULL;
214
215	mutex_unlock(&con->page_retirement_lock);
216}
217
218static int amdgpu_umc_do_page_retirement(struct amdgpu_device *adev,
219		void *ras_error_status,
220		struct amdgpu_iv_entry *entry,
221		uint32_t reset)
222{
223	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
224	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
225
226	kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
227	amdgpu_umc_handle_bad_pages(adev, ras_error_status);
228
229	if ((err_data->ue_count || err_data->de_count) &&
230	    (reset || amdgpu_ras_is_rma(adev))) {
231		con->gpu_reset_flags |= reset;
232		amdgpu_ras_reset_gpu(adev);
233	}
234
235	return AMDGPU_RAS_SUCCESS;
236}
237
238int amdgpu_umc_pasid_poison_handler(struct amdgpu_device *adev,
239			enum amdgpu_ras_block block, uint16_t pasid,
240			pasid_notify pasid_fn, void *data, uint32_t reset)
241{
242	int ret = AMDGPU_RAS_SUCCESS;
243
244	if (adev->gmc.xgmi.connected_to_cpu ||
245		adev->gmc.is_app_apu) {
246		if (reset) {
247			/* MCA poison handler is only responsible for GPU reset,
248			 * let MCA notifier do page retirement.
249			 */
250			kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
251			amdgpu_ras_reset_gpu(adev);
252		}
253		return ret;
254	}
255
256	if (!amdgpu_sriov_vf(adev)) {
257		if (amdgpu_ip_version(adev, UMC_HWIP, 0) < IP_VERSION(12, 0, 0)) {
258			struct ras_err_data err_data;
259			struct ras_common_if head = {
260				.block = AMDGPU_RAS_BLOCK__UMC,
261			};
262			struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head);
263
264			ret = amdgpu_ras_error_data_init(&err_data);
265			if (ret)
266				return ret;
267
268			ret = amdgpu_umc_do_page_retirement(adev, &err_data, NULL, reset);
269
270			if (ret == AMDGPU_RAS_SUCCESS && obj) {
271				obj->err_data.ue_count += err_data.ue_count;
272				obj->err_data.ce_count += err_data.ce_count;
273				obj->err_data.de_count += err_data.de_count;
274			}
275
276			amdgpu_ras_error_data_fini(&err_data);
277		} else if (amdgpu_uniras_enabled(adev)) {
278			struct ras_ih_info ih_info = {0};
279
280			ih_info.block = block;
281			ih_info.pasid = pasid;
282			ih_info.reset = reset;
283			ih_info.pasid_fn = pasid_fn;
284			ih_info.data = data;
285			amdgpu_ras_mgr_handle_consumer_interrupt(adev, &ih_info);
286		} else {
287			struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
288			int ret;
289
290			ret = amdgpu_ras_put_poison_req(adev,
291				block, pasid, pasid_fn, data, reset);
292			if (!ret) {
293				atomic_inc(&con->page_retirement_req_cnt);
294				atomic_inc(&con->poison_consumption_count);
295				wake_up(&con->page_retirement_wq);
296			}
297		}
298	} else {
299		if (adev->virt.ops && adev->virt.ops->ras_poison_handler)
300			adev->virt.ops->ras_poison_handler(adev, block);
301		else
302			dev_warn(adev->dev,
303				"No ras_poison_handler interface in SRIOV!\n");
304	}
305
306	return ret;
307}
308
309int amdgpu_umc_poison_handler(struct amdgpu_device *adev,
310			enum amdgpu_ras_block block, uint32_t reset)
311{
312	return amdgpu_umc_pasid_poison_handler(adev,
313				block, 0, NULL, NULL, reset);
314}
315
316int amdgpu_umc_process_ras_data_cb(struct amdgpu_device *adev,
317		void *ras_error_status,
318		struct amdgpu_iv_entry *entry)
319{
320	return amdgpu_umc_do_page_retirement(adev, ras_error_status, entry,
321				AMDGPU_RAS_GPU_RESET_MODE1_RESET);
322}
323
324int amdgpu_umc_ras_sw_init(struct amdgpu_device *adev)
325{
326	int err;
327	struct amdgpu_umc_ras *ras;
328
329	if (!adev->umc.ras)
330		return 0;
331
332	ras = adev->umc.ras;
333
334	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
335	if (err) {
336		dev_err(adev->dev, "Failed to register umc ras block!\n");
337		return err;
338	}
339
340	strcpy(adev->umc.ras->ras_block.ras_comm.name, "umc");
341	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__UMC;
342	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
343	adev->umc.ras_if = &ras->ras_block.ras_comm;
344
345	if (!ras->ras_block.ras_late_init)
346		ras->ras_block.ras_late_init = amdgpu_umc_ras_late_init;
347
348	if (!ras->ras_block.ras_cb)
349		ras->ras_block.ras_cb = amdgpu_umc_process_ras_data_cb;
350
351	return 0;
352}
353
354int amdgpu_umc_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
355{
356	int r;
357
358	r = amdgpu_ras_block_late_init(adev, ras_block);
359	if (r)
360		return r;
361
362	if (amdgpu_sriov_vf(adev))
363		return r;
364
365	if (amdgpu_ras_is_supported(adev, ras_block->block)) {
366		r = amdgpu_irq_get(adev, &adev->gmc.ecc_irq, 0);
367		if (r)
368			goto late_fini;
369	}
370
371	/* ras init of specific umc version */
372	if (adev->umc.ras &&
373	    adev->umc.ras->err_cnt_init)
374		adev->umc.ras->err_cnt_init(adev);
375
376	return 0;
377
378late_fini:
379	amdgpu_ras_block_late_fini(adev, ras_block);
380	return r;
381}
382
383int amdgpu_umc_process_ecc_irq(struct amdgpu_device *adev,
384		struct amdgpu_irq_src *source,
385		struct amdgpu_iv_entry *entry)
386{
387	struct ras_common_if *ras_if = adev->umc.ras_if;
388	struct ras_dispatch_if ih_data = {
389		.entry = entry,
390	};
391
392	if (!ras_if)
393		return 0;
394
395	ih_data.head = *ras_if;
396
397	amdgpu_ras_interrupt_dispatch(adev, &ih_data);
398	return 0;
399}
400
401int amdgpu_umc_fill_error_record(struct ras_err_data *err_data,
402		uint64_t err_addr,
403		uint64_t retired_page,
404		uint32_t channel_index,
405		uint32_t umc_inst)
406{
407	struct eeprom_table_record *err_rec;
408
409	if (!err_data ||
410	    !err_data->err_addr ||
411	    (err_data->err_addr_cnt >= err_data->err_addr_len))
412		return -EINVAL;
413
414	err_rec = &err_data->err_addr[err_data->err_addr_cnt];
415
416	err_rec->address = err_addr;
417	/* page frame address is saved */
418	err_rec->retired_page = retired_page >> AMDGPU_GPU_PAGE_SHIFT;
419	err_rec->ts = (uint64_t)ktime_get_real_seconds();
420	err_rec->err_type = AMDGPU_RAS_EEPROM_ERR_NON_RECOVERABLE;
421	err_rec->cu = 0;
422	err_rec->mem_channel = channel_index;
423	err_rec->mcumc_id = umc_inst;
424
425	err_data->err_addr_cnt++;
426
427	return 0;
428}
429
430static int amdgpu_umc_loop_all_aid(struct amdgpu_device *adev, umc_func func,
431				   void *data)
432{
433	uint32_t umc_node_inst;
434	uint32_t node_inst;
435	uint32_t umc_inst;
436	uint32_t ch_inst;
437	int ret;
438
439	/*
440	 * This loop is done based on the following -
441	 * umc.active mask = mask of active umc instances across all nodes
442	 * umc.umc_inst_num = maximum number of umc instancess per node
443	 * umc.node_inst_num = maximum number of node instances
444	 * Channel instances are not assumed to be harvested.
445	 */
446	dev_dbg(adev->dev, "active umcs :%lx umc_inst per node: %d",
447		adev->umc.active_mask, adev->umc.umc_inst_num);
448	for_each_set_bit(umc_node_inst, &(adev->umc.active_mask),
449			 adev->umc.node_inst_num * adev->umc.umc_inst_num) {
450		node_inst = umc_node_inst / adev->umc.umc_inst_num;
451		umc_inst = umc_node_inst % adev->umc.umc_inst_num;
452		LOOP_UMC_CH_INST(ch_inst) {
453			dev_dbg(adev->dev,
454				"node_inst :%d umc_inst: %d ch_inst: %d",
455				node_inst, umc_inst, ch_inst);
456			ret = func(adev, node_inst, umc_inst, ch_inst, data);
457			if (ret) {
458				dev_err(adev->dev,
459					"Node %d umc %d ch %d func returns %d\n",
460					node_inst, umc_inst, ch_inst, ret);
461				return ret;
462			}
463		}
464	}
465
466	return 0;
467}
468
469int amdgpu_umc_loop_channels(struct amdgpu_device *adev,
470			umc_func func, void *data)
471{
472	uint32_t node_inst       = 0;
473	uint32_t umc_inst        = 0;
474	uint32_t ch_inst         = 0;
475	int ret = 0;
476
477	if (adev->aid_mask)
478		return amdgpu_umc_loop_all_aid(adev, func, data);
479
480	if (adev->umc.node_inst_num) {
481		LOOP_UMC_EACH_NODE_INST_AND_CH(node_inst, umc_inst, ch_inst) {
482			ret = func(adev, node_inst, umc_inst, ch_inst, data);
483			if (ret) {
484				dev_err(adev->dev, "Node %d umc %d ch %d func returns %d\n",
485					node_inst, umc_inst, ch_inst, ret);
486				return ret;
487			}
488		}
489	} else {
490		LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) {
491			ret = func(adev, 0, umc_inst, ch_inst, data);
492			if (ret) {
493				dev_err(adev->dev, "Umc %d ch %d func returns %d\n",
494					umc_inst, ch_inst, ret);
495				return ret;
496			}
497		}
498	}
499
500	return 0;
501}
502
503int amdgpu_umc_update_ecc_status(struct amdgpu_device *adev,
504				uint64_t status, uint64_t ipid, uint64_t addr)
505{
506	if (adev->umc.ras->update_ecc_status)
507		return adev->umc.ras->update_ecc_status(adev,
508					status, ipid, addr);
509	return 0;
510}
511
512int amdgpu_umc_logs_ecc_err(struct amdgpu_device *adev,
513		struct radix_tree_root *ecc_tree, struct ras_ecc_err *ecc_err)
514{
515	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
516	struct ras_ecc_log_info *ecc_log;
517	int ret;
518
519	ecc_log = &con->umc_ecc_log;
520
521	mutex_lock(&ecc_log->lock);
522	ret = radix_tree_insert(ecc_tree, ecc_err->pa_pfn, ecc_err);
523	if (!ret)
524		radix_tree_tag_set(ecc_tree,
525			ecc_err->pa_pfn, UMC_ECC_NEW_DETECTED_TAG);
526	mutex_unlock(&ecc_log->lock);
527
528	return ret;
529}
530
531int amdgpu_umc_pages_in_a_row(struct amdgpu_device *adev,
532			struct ras_err_data *err_data, uint64_t pa_addr)
533{
534	struct ta_ras_query_address_output addr_out;
535
536	/* reinit err_data */
537	err_data->err_addr_cnt = 0;
538	err_data->err_addr_len = adev->umc.retire_unit;
539
540	addr_out.pa.pa = pa_addr;
541	if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
542		return adev->umc.ras->convert_ras_err_addr(adev, err_data, NULL,
543				&addr_out, false);
544	else
545		return -EINVAL;
546}
547
548int amdgpu_umc_lookup_bad_pages_in_a_row(struct amdgpu_device *adev,
549			uint64_t pa_addr, uint64_t *pfns, int len)
550{
551	int i, ret;
552	struct ras_err_data err_data;
553
554	err_data.err_addr = kcalloc(adev->umc.retire_unit,
555				sizeof(struct eeprom_table_record), GFP_KERNEL);
556	if (!err_data.err_addr) {
557		dev_warn(adev->dev, "Failed to alloc memory in bad page lookup!\n");
558		return 0;
559	}
560
561	ret = amdgpu_umc_pages_in_a_row(adev, &err_data, pa_addr);
562	if (ret)
563		goto out;
564
565	for (i = 0; i < adev->umc.retire_unit; i++) {
566		if (i >= len)
567			goto out;
568
569		pfns[i] = err_data.err_addr[i].retired_page;
570	}
571	ret = i;
572	adev->umc.err_addr_cnt = err_data.err_addr_cnt;
573
574out:
575	kfree(err_data.err_addr);
576	return ret;
577}
578
579int amdgpu_umc_mca_to_addr(struct amdgpu_device *adev,
580			uint64_t err_addr, uint32_t ch, uint32_t umc,
581			uint32_t node, uint32_t socket,
582			struct ta_ras_query_address_output *addr_out, bool dump_addr)
583{
584	struct ta_ras_query_address_input addr_in;
585	int ret;
586
587	memset(&addr_in, 0, sizeof(addr_in));
588	addr_in.ma.err_addr = err_addr;
589	addr_in.ma.ch_inst = ch;
590	addr_in.ma.umc_inst = umc;
591	addr_in.ma.node_inst = node;
592	addr_in.ma.socket_id = socket;
593
594	if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) {
595		ret = adev->umc.ras->convert_ras_err_addr(adev, NULL, &addr_in,
596				addr_out, dump_addr);
597		if (ret)
598			return ret;
599	} else {
600		return 0;
601	}
602
603	return 0;
604}
605
606int amdgpu_umc_pa2mca(struct amdgpu_device *adev,
607		uint64_t pa, uint64_t *mca, enum amdgpu_memory_partition nps)
608{
609	struct ta_ras_query_address_input addr_in;
610	struct ta_ras_query_address_output addr_out;
611	int ret;
612
613	/* nps: the pa belongs to */
614	addr_in.pa.pa = pa | ((uint64_t)nps << 58);
615	addr_in.addr_type = TA_RAS_PA_TO_MCA;
616	ret = psp_ras_query_address(&adev->psp, &addr_in, &addr_out);
617	if (ret) {
618		dev_warn(adev->dev, "Failed to query RAS MCA address for 0x%llx",
619			pa);
620
621		return ret;
622	}
623
624	*mca = addr_out.ma.err_addr;
625
626	return 0;
627}