sifive_ccache.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * SiFive composable cache controller Driver
  4 *
  5 * Copyright (C) 2018-2022 SiFive, Inc.
  6 *
  7 */
  8
  9#define pr_fmt(fmt) "CCACHE: " fmt
 10
 11#include <linux/align.h>
 12#include <linux/debugfs.h>
 13#include <linux/interrupt.h>
 14#include <linux/of_irq.h>
 15#include <linux/of_address.h>
 16#include <linux/device.h>
 17#include <linux/bitfield.h>
 18#include <linux/platform_device.h>
 19#include <linux/property.h>
 20#include <asm/cacheflush.h>
 21#include <asm/cacheinfo.h>
 22#include <asm/dma-noncoherent.h>
 23#include <soc/sifive/sifive_ccache.h>
 24
 25#define SIFIVE_CCACHE_DIRECCFIX_LOW 0x100
 26#define SIFIVE_CCACHE_DIRECCFIX_HIGH 0x104
 27#define SIFIVE_CCACHE_DIRECCFIX_COUNT 0x108
 28
 29#define SIFIVE_CCACHE_DIRECCFAIL_LOW 0x120
 30#define SIFIVE_CCACHE_DIRECCFAIL_HIGH 0x124
 31#define SIFIVE_CCACHE_DIRECCFAIL_COUNT 0x128
 32
 33#define SIFIVE_CCACHE_DATECCFIX_LOW 0x140
 34#define SIFIVE_CCACHE_DATECCFIX_HIGH 0x144
 35#define SIFIVE_CCACHE_DATECCFIX_COUNT 0x148
 36
 37#define SIFIVE_CCACHE_DATECCFAIL_LOW 0x160
 38#define SIFIVE_CCACHE_DATECCFAIL_HIGH 0x164
 39#define SIFIVE_CCACHE_DATECCFAIL_COUNT 0x168
 40
 41#define SIFIVE_CCACHE_CONFIG 0x00
 42#define SIFIVE_CCACHE_CONFIG_BANK_MASK GENMASK_ULL(7, 0)
 43#define SIFIVE_CCACHE_CONFIG_WAYS_MASK GENMASK_ULL(15, 8)
 44#define SIFIVE_CCACHE_CONFIG_SETS_MASK GENMASK_ULL(23, 16)
 45#define SIFIVE_CCACHE_CONFIG_BLKS_MASK GENMASK_ULL(31, 24)
 46
 47#define SIFIVE_CCACHE_FLUSH64 0x200
 48#define SIFIVE_CCACHE_FLUSH32 0x240
 49
 50#define SIFIVE_CCACHE_WAYENABLE 0x08
 51#define SIFIVE_CCACHE_ECCINJECTERR 0x40
 52
 53#define SIFIVE_CCACHE_MAX_ECCINTR 4
 54#define SIFIVE_CCACHE_LINE_SIZE 64
 55
 56static void __iomem *ccache_base;
 57static int g_irq[SIFIVE_CCACHE_MAX_ECCINTR];
 58static struct riscv_cacheinfo_ops ccache_cache_ops;
 59static int level;
 60
 61enum {
 62	DIR_CORR = 0,
 63	DATA_CORR,
 64	DATA_UNCORR,
 65	DIR_UNCORR,
 66};
 67
 68enum {
 69	QUIRK_NONSTANDARD_CACHE_OPS	= BIT(0),
 70	QUIRK_BROKEN_DATA_UNCORR	= BIT(1),
 71};
 72
 73#ifdef CONFIG_DEBUG_FS
 74static struct dentry *sifive_test;
 75
 76static ssize_t ccache_write(struct file *file, const char __user *data,
 77			    size_t count, loff_t *ppos)
 78{
 79	unsigned int val;
 80
 81	if (kstrtouint_from_user(data, count, 0, &val))
 82		return -EINVAL;
 83	if ((val < 0xFF) || (val >= 0x10000 && val < 0x100FF))
 84		writel(val, ccache_base + SIFIVE_CCACHE_ECCINJECTERR);
 85	else
 86		return -EINVAL;
 87	return count;
 88}
 89
 90static const struct file_operations ccache_fops = {
 91	.owner = THIS_MODULE,
 92	.open = simple_open,
 93	.write = ccache_write
 94};
 95
 96static void setup_sifive_debug(void)
 97{
 98	sifive_test = debugfs_create_dir("sifive_ccache_cache", NULL);
 99
100	debugfs_create_file("sifive_debug_inject_error", 0200,
101			    sifive_test, NULL, &ccache_fops);
102}
103#endif
104
105static void ccache_config_read(void)
106{
107	u32 cfg;
108
109	cfg = readl(ccache_base + SIFIVE_CCACHE_CONFIG);
110	pr_info("%llu banks, %llu ways, sets/bank=%llu, bytes/block=%llu\n",
111		FIELD_GET(SIFIVE_CCACHE_CONFIG_BANK_MASK, cfg),
112		FIELD_GET(SIFIVE_CCACHE_CONFIG_WAYS_MASK, cfg),
113		BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_SETS_MASK, cfg)),
114		BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_BLKS_MASK, cfg)));
115
116	cfg = readl(ccache_base + SIFIVE_CCACHE_WAYENABLE);
117	pr_info("Index of the largest way enabled: %u\n", cfg);
118}
119
120static const struct of_device_id sifive_ccache_ids[] = {
121	{ .compatible = "eswin,eic7700-l3-cache",
122	  .data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS) },
123	{ .compatible = "sifive,fu540-c000-ccache" },
124	{ .compatible = "sifive,fu740-c000-ccache" },
125	{ .compatible = "starfive,jh7100-ccache",
126	  .data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS | QUIRK_BROKEN_DATA_UNCORR) },
127	{ .compatible = "sifive,ccache0" },
128	{ /* end of table */ }
129};
130
131static ATOMIC_NOTIFIER_HEAD(ccache_err_chain);
132
133int register_sifive_ccache_error_notifier(struct notifier_block *nb)
134{
135	return atomic_notifier_chain_register(&ccache_err_chain, nb);
136}
137EXPORT_SYMBOL_GPL(register_sifive_ccache_error_notifier);
138
139int unregister_sifive_ccache_error_notifier(struct notifier_block *nb)
140{
141	return atomic_notifier_chain_unregister(&ccache_err_chain, nb);
142}
143EXPORT_SYMBOL_GPL(unregister_sifive_ccache_error_notifier);
144
145#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
146static void ccache_flush_range(phys_addr_t start, size_t len)
147{
148	phys_addr_t end = start + len;
149	phys_addr_t line;
150
151	if (!len)
152		return;
153
154	mb(); /* complete earlier memory accesses before the cache flush */
155	for (line = ALIGN_DOWN(start, SIFIVE_CCACHE_LINE_SIZE); line < end;
156			line += SIFIVE_CCACHE_LINE_SIZE) {
157#ifdef CONFIG_32BIT
158		writel_relaxed(line >> 4, ccache_base + SIFIVE_CCACHE_FLUSH32);
159#else
160		writeq_relaxed(line, ccache_base + SIFIVE_CCACHE_FLUSH64);
161#endif
162	}
163	mb(); /* issue later memory accesses after the cache flush */
164}
165
166static const struct riscv_nonstd_cache_ops ccache_mgmt_ops __initconst = {
167	.wback = &ccache_flush_range,
168	.inv = &ccache_flush_range,
169	.wback_inv = &ccache_flush_range,
170};
171#endif /* CONFIG_RISCV_NONSTANDARD_CACHE_OPS */
172
173static int ccache_largest_wayenabled(void)
174{
175	return readl(ccache_base + SIFIVE_CCACHE_WAYENABLE) & 0xFF;
176}
177
178static ssize_t number_of_ways_enabled_show(struct device *dev,
179					   struct device_attribute *attr,
180					   char *buf)
181{
182	return sprintf(buf, "%u\n", ccache_largest_wayenabled());
183}
184
185static DEVICE_ATTR_RO(number_of_ways_enabled);
186
187static struct attribute *priv_attrs[] = {
188	&dev_attr_number_of_ways_enabled.attr,
189	NULL,
190};
191
192static const struct attribute_group priv_attr_group = {
193	.attrs = priv_attrs,
194};
195
196static const struct attribute_group *ccache_get_priv_group(struct cacheinfo
197							   *this_leaf)
198{
199	/* We want to use private group for composable cache only */
200	if (this_leaf->level == level)
201		return &priv_attr_group;
202	else
203		return NULL;
204}
205
206static irqreturn_t ccache_int_handler(int irq, void *device)
207{
208	unsigned int add_h, add_l;
209
210	if (irq == g_irq[DIR_CORR]) {
211		add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_HIGH);
212		add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_LOW);
213		pr_err("DirError @ 0x%08X.%08X\n", add_h, add_l);
214		/* Reading this register clears the DirError interrupt sig */
215		readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_COUNT);
216		atomic_notifier_call_chain(&ccache_err_chain,
217					   SIFIVE_CCACHE_ERR_TYPE_CE,
218					   "DirECCFix");
219	}
220	if (irq == g_irq[DIR_UNCORR]) {
221		add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_HIGH);
222		add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_LOW);
223		/* Reading this register clears the DirFail interrupt sig */
224		readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_COUNT);
225		atomic_notifier_call_chain(&ccache_err_chain,
226					   SIFIVE_CCACHE_ERR_TYPE_UE,
227					   "DirECCFail");
228		panic("CCACHE: DirFail @ 0x%08X.%08X\n", add_h, add_l);
229	}
230	if (irq == g_irq[DATA_CORR]) {
231		add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_HIGH);
232		add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_LOW);
233		pr_err("DataError @ 0x%08X.%08X\n", add_h, add_l);
234		/* Reading this register clears the DataError interrupt sig */
235		readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_COUNT);
236		atomic_notifier_call_chain(&ccache_err_chain,
237					   SIFIVE_CCACHE_ERR_TYPE_CE,
238					   "DatECCFix");
239	}
240	if (irq == g_irq[DATA_UNCORR]) {
241		add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_HIGH);
242		add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_LOW);
243		pr_err("DataFail @ 0x%08X.%08X\n", add_h, add_l);
244		/* Reading this register clears the DataFail interrupt sig */
245		readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_COUNT);
246		atomic_notifier_call_chain(&ccache_err_chain,
247					   SIFIVE_CCACHE_ERR_TYPE_UE,
248					   "DatECCFail");
249	}
250
251	return IRQ_HANDLED;
252}
253
254static int sifive_ccache_probe(struct platform_device *pdev)
255{
256	struct device *dev = &pdev->dev;
257	unsigned long quirks;
258	int intr_num, rc;
259
260	quirks = (unsigned long)device_get_match_data(dev);
261
262	intr_num = platform_irq_count(pdev);
263	if (!intr_num)
264		return dev_err_probe(dev, -ENODEV, "No interrupts property\n");
265
266	for (int i = 0; i < intr_num; i++) {
267		if (i == DATA_UNCORR && (quirks & QUIRK_BROKEN_DATA_UNCORR))
268			continue;
269
270		g_irq[i] = platform_get_irq(pdev, i);
271		if (g_irq[i] < 0)
272			return g_irq[i];
273
274		rc = devm_request_irq(dev, g_irq[i], ccache_int_handler, 0, "ccache_ecc", NULL);
275		if (rc)
276			return dev_err_probe(dev, rc, "Could not request IRQ %d\n", g_irq[i]);
277	}
278
279	return 0;
280}
281
282static struct platform_driver sifive_ccache_driver = {
283	.probe	= sifive_ccache_probe,
284	.driver	= {
285		.name		= "sifive_ccache",
286		.of_match_table	= sifive_ccache_ids,
287	},
288};
289
290static int __init sifive_ccache_init(void)
291{
292	struct device_node *np;
293	struct resource res;
294	const struct of_device_id *match;
295	unsigned long quirks __maybe_unused;
296	int rc;
297
298	np = of_find_matching_node_and_match(NULL, sifive_ccache_ids, &match);
299	if (!np)
300		return -ENODEV;
301
302	quirks = (uintptr_t)match->data;
303
304	if (of_address_to_resource(np, 0, &res)) {
305		rc = -ENODEV;
306		goto err_node_put;
307	}
308
309	ccache_base = ioremap(res.start, resource_size(&res));
310	if (!ccache_base) {
311		rc = -ENOMEM;
312		goto err_node_put;
313	}
314
315	if (of_property_read_u32(np, "cache-level", &level)) {
316		rc = -ENOENT;
317		goto err_unmap;
318	}
319
320#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
321	if (quirks & QUIRK_NONSTANDARD_CACHE_OPS) {
322		riscv_cbom_block_size = SIFIVE_CCACHE_LINE_SIZE;
323		riscv_noncoherent_supported();
324		riscv_noncoherent_register_cache_ops(&ccache_mgmt_ops);
325	}
326#endif
327
328	ccache_config_read();
329
330	ccache_cache_ops.get_priv_group = ccache_get_priv_group;
331	riscv_set_cacheinfo_ops(&ccache_cache_ops);
332
333#ifdef CONFIG_DEBUG_FS
334	setup_sifive_debug();
335#endif
336
337	rc = platform_driver_register(&sifive_ccache_driver);
338	if (rc)
339		goto err_unmap;
340
341	of_node_put(np);
342
343	return 0;
344
345err_unmap:
346	iounmap(ccache_base);
347err_node_put:
348	of_node_put(np);
349	return rc;
350}
351
352arch_initcall(sifive_ccache_init);