ref_tracker.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2
  3#define pr_fmt(fmt) "ref_tracker: " fmt
  4
  5#include <linux/export.h>
  6#include <linux/list_sort.h>
  7#include <linux/ref_tracker.h>
  8#include <linux/slab.h>
  9#include <linux/stacktrace.h>
 10#include <linux/stackdepot.h>
 11#include <linux/seq_file.h>
 12
 13#define REF_TRACKER_STACK_ENTRIES 16
 14#define STACK_BUF_SIZE 1024
 15
 16struct ref_tracker {
 17	struct list_head	head;   /* anchor into dir->list or dir->quarantine */
 18	bool			dead;
 19	depot_stack_handle_t	alloc_stack_handle;
 20	depot_stack_handle_t	free_stack_handle;
 21};
 22
 23struct ref_tracker_dir_stats {
 24	int total;
 25	int count;
 26	struct {
 27		depot_stack_handle_t stack_handle;
 28		unsigned int count;
 29	} stacks[];
 30};
 31
 32#ifdef CONFIG_DEBUG_FS
 33#include <linux/xarray.h>
 34
 35/*
 36 * ref_tracker_dir_init() is usually called in allocation-safe contexts, but
 37 * the same is not true of ref_tracker_dir_exit() which can be called from
 38 * anywhere an object is freed. Removing debugfs dentries is a blocking
 39 * operation, so we defer that work to the debugfs_reap_worker.
 40 *
 41 * Each dentry is tracked in the appropriate xarray.  When
 42 * ref_tracker_dir_exit() is called, its entries in the xarrays are marked and
 43 * the workqueue job is scheduled. The worker then runs and deletes any marked
 44 * dentries asynchronously.
 45 */
 46static struct xarray		debugfs_dentries;
 47static struct xarray		debugfs_symlinks;
 48static struct work_struct	debugfs_reap_worker;
 49
 50#define REF_TRACKER_DIR_DEAD	XA_MARK_0
 51static inline void ref_tracker_debugfs_mark(struct ref_tracker_dir *dir)
 52{
 53	unsigned long flags;
 54
 55	xa_lock_irqsave(&debugfs_dentries, flags);
 56	__xa_set_mark(&debugfs_dentries, (unsigned long)dir, REF_TRACKER_DIR_DEAD);
 57	xa_unlock_irqrestore(&debugfs_dentries, flags);
 58
 59	xa_lock_irqsave(&debugfs_symlinks, flags);
 60	__xa_set_mark(&debugfs_symlinks, (unsigned long)dir, REF_TRACKER_DIR_DEAD);
 61	xa_unlock_irqrestore(&debugfs_symlinks, flags);
 62
 63	schedule_work(&debugfs_reap_worker);
 64}
 65#else
 66static inline void ref_tracker_debugfs_mark(struct ref_tracker_dir *dir)
 67{
 68}
 69#endif
 70
 71static struct ref_tracker_dir_stats *
 72ref_tracker_get_stats(struct ref_tracker_dir *dir, unsigned int limit)
 73{
 74	struct ref_tracker_dir_stats *stats;
 75	struct ref_tracker *tracker;
 76
 77	stats = kmalloc(struct_size(stats, stacks, limit),
 78			GFP_NOWAIT);
 79	if (!stats)
 80		return ERR_PTR(-ENOMEM);
 81	stats->total = 0;
 82	stats->count = 0;
 83
 84	list_for_each_entry(tracker, &dir->list, head) {
 85		depot_stack_handle_t stack = tracker->alloc_stack_handle;
 86		int i;
 87
 88		++stats->total;
 89		for (i = 0; i < stats->count; ++i)
 90			if (stats->stacks[i].stack_handle == stack)
 91				break;
 92		if (i >= limit)
 93			continue;
 94		if (i >= stats->count) {
 95			stats->stacks[i].stack_handle = stack;
 96			stats->stacks[i].count = 0;
 97			++stats->count;
 98		}
 99		++stats->stacks[i].count;
100	}
101
102	return stats;
103}
104
105struct ostream {
106	void __ostream_printf (*func)(struct ostream *stream, char *fmt, ...);
107	char *prefix;
108	char *buf;
109	struct seq_file *seq;
110	int size, used;
111};
112
113static void __ostream_printf pr_ostream_log(struct ostream *stream, char *fmt, ...)
114{
115	va_list args;
116
117	va_start(args, fmt);
118	vprintk(fmt, args);
119	va_end(args);
120}
121
122static void __ostream_printf pr_ostream_buf(struct ostream *stream, char *fmt, ...)
123{
124	int ret, len = stream->size - stream->used;
125	va_list args;
126
127	va_start(args, fmt);
128	ret = vsnprintf(stream->buf + stream->used, len, fmt, args);
129	va_end(args);
130	if (ret > 0)
131		stream->used += min(ret, len);
132}
133
134#define pr_ostream(stream, fmt, args...) \
135({ \
136	struct ostream *_s = (stream); \
137\
138	_s->func(_s, fmt, ##args); \
139})
140
141static void
142__ref_tracker_dir_pr_ostream(struct ref_tracker_dir *dir,
143			     unsigned int display_limit, struct ostream *s)
144{
145	struct ref_tracker_dir_stats *stats;
146	unsigned int i = 0, skipped;
147	depot_stack_handle_t stack;
148	char *sbuf;
149
150	lockdep_assert_held(&dir->lock);
151
152	if (list_empty(&dir->list))
153		return;
154
155	stats = ref_tracker_get_stats(dir, display_limit);
156	if (IS_ERR(stats)) {
157		pr_ostream(s, "%s%s@%p: couldn't get stats, error %pe\n",
158			   s->prefix, dir->class, dir, stats);
159		return;
160	}
161
162	sbuf = kmalloc(STACK_BUF_SIZE, GFP_NOWAIT);
163
164	for (i = 0, skipped = stats->total; i < stats->count; ++i) {
165		stack = stats->stacks[i].stack_handle;
166		if (sbuf && !stack_depot_snprint(stack, sbuf, STACK_BUF_SIZE, 4))
167			sbuf[0] = 0;
168		pr_ostream(s, "%s%s@%p has %d/%d users at\n%s\n", s->prefix,
169			   dir->class, dir, stats->stacks[i].count,
170			   stats->total, sbuf);
171		skipped -= stats->stacks[i].count;
172	}
173
174	if (skipped)
175		pr_ostream(s, "%s%s@%p skipped reports about %d/%d users.\n",
176			   s->prefix, dir->class, dir, skipped, stats->total);
177
178	kfree(sbuf);
179
180	kfree(stats);
181}
182
183void ref_tracker_dir_print_locked(struct ref_tracker_dir *dir,
184				  unsigned int display_limit)
185{
186	struct ostream os = { .func = pr_ostream_log,
187			      .prefix = "ref_tracker: " };
188
189	__ref_tracker_dir_pr_ostream(dir, display_limit, &os);
190}
191EXPORT_SYMBOL(ref_tracker_dir_print_locked);
192
193void ref_tracker_dir_print(struct ref_tracker_dir *dir,
194			   unsigned int display_limit)
195{
196	unsigned long flags;
197
198	spin_lock_irqsave(&dir->lock, flags);
199	ref_tracker_dir_print_locked(dir, display_limit);
200	spin_unlock_irqrestore(&dir->lock, flags);
201}
202EXPORT_SYMBOL(ref_tracker_dir_print);
203
204int ref_tracker_dir_snprint(struct ref_tracker_dir *dir, char *buf, size_t size)
205{
206	struct ostream os = { .func = pr_ostream_buf,
207			      .prefix = "ref_tracker: ",
208			      .buf = buf,
209			      .size = size };
210	unsigned long flags;
211
212	spin_lock_irqsave(&dir->lock, flags);
213	__ref_tracker_dir_pr_ostream(dir, 16, &os);
214	spin_unlock_irqrestore(&dir->lock, flags);
215
216	return os.used;
217}
218EXPORT_SYMBOL(ref_tracker_dir_snprint);
219
220void ref_tracker_dir_exit(struct ref_tracker_dir *dir)
221{
222	struct ref_tracker *tracker, *n;
223	unsigned long flags;
224	bool leak = false;
225
226	dir->dead = true;
227	/*
228	 * The xarray entries must be marked before the dir->lock is taken to
229	 * protect simultaneous debugfs readers.
230	 */
231	ref_tracker_debugfs_mark(dir);
232	spin_lock_irqsave(&dir->lock, flags);
233	list_for_each_entry_safe(tracker, n, &dir->quarantine, head) {
234		list_del(&tracker->head);
235		kfree(tracker);
236		dir->quarantine_avail++;
237	}
238	if (!list_empty(&dir->list)) {
239		ref_tracker_dir_print_locked(dir, 16);
240		leak = true;
241		list_for_each_entry_safe(tracker, n, &dir->list, head) {
242			list_del(&tracker->head);
243			kfree(tracker);
244		}
245	}
246	spin_unlock_irqrestore(&dir->lock, flags);
247	WARN_ON_ONCE(leak);
248	WARN_ON_ONCE(refcount_read(&dir->untracked) != 1);
249	WARN_ON_ONCE(refcount_read(&dir->no_tracker) != 1);
250}
251EXPORT_SYMBOL(ref_tracker_dir_exit);
252
253int ref_tracker_alloc(struct ref_tracker_dir *dir,
254		      struct ref_tracker **trackerp,
255		      gfp_t gfp)
256{
257	unsigned long entries[REF_TRACKER_STACK_ENTRIES];
258	struct ref_tracker *tracker;
259	unsigned int nr_entries;
260	gfp_t gfp_mask = gfp | __GFP_NOWARN;
261	unsigned long flags;
262
263	WARN_ON_ONCE(dir->dead);
264
265	if (!trackerp) {
266		refcount_inc(&dir->no_tracker);
267		return 0;
268	}
269	if (gfp & __GFP_DIRECT_RECLAIM)
270		gfp_mask |= __GFP_NOFAIL;
271	*trackerp = tracker = kzalloc(sizeof(*tracker), gfp_mask);
272	if (unlikely(!tracker)) {
273		pr_err_once("memory allocation failure, unreliable refcount tracker.\n");
274		refcount_inc(&dir->untracked);
275		return -ENOMEM;
276	}
277	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
278	tracker->alloc_stack_handle = stack_depot_save(entries, nr_entries, gfp);
279
280	spin_lock_irqsave(&dir->lock, flags);
281	list_add(&tracker->head, &dir->list);
282	spin_unlock_irqrestore(&dir->lock, flags);
283	return 0;
284}
285EXPORT_SYMBOL_GPL(ref_tracker_alloc);
286
287int ref_tracker_free(struct ref_tracker_dir *dir,
288		     struct ref_tracker **trackerp)
289{
290	unsigned long entries[REF_TRACKER_STACK_ENTRIES];
291	depot_stack_handle_t stack_handle;
292	struct ref_tracker *tracker;
293	unsigned int nr_entries;
294	unsigned long flags;
295
296	WARN_ON_ONCE(dir->dead);
297
298	if (!trackerp) {
299		refcount_dec(&dir->no_tracker);
300		return 0;
301	}
302	tracker = *trackerp;
303	if (!tracker) {
304		refcount_dec(&dir->untracked);
305		return -EEXIST;
306	}
307	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
308	stack_handle = stack_depot_save(entries, nr_entries,
309					GFP_NOWAIT);
310
311	spin_lock_irqsave(&dir->lock, flags);
312	if (tracker->dead) {
313		pr_err("reference already released.\n");
314		if (tracker->alloc_stack_handle) {
315			pr_err("allocated in:\n");
316			stack_depot_print(tracker->alloc_stack_handle);
317		}
318		if (tracker->free_stack_handle) {
319			pr_err("freed in:\n");
320			stack_depot_print(tracker->free_stack_handle);
321		}
322		spin_unlock_irqrestore(&dir->lock, flags);
323		WARN_ON_ONCE(1);
324		return -EINVAL;
325	}
326	tracker->dead = true;
327
328	tracker->free_stack_handle = stack_handle;
329
330	list_move_tail(&tracker->head, &dir->quarantine);
331	if (!dir->quarantine_avail) {
332		tracker = list_first_entry(&dir->quarantine, struct ref_tracker, head);
333		list_del(&tracker->head);
334	} else {
335		dir->quarantine_avail--;
336		tracker = NULL;
337	}
338	spin_unlock_irqrestore(&dir->lock, flags);
339
340	kfree(tracker);
341	return 0;
342}
343EXPORT_SYMBOL_GPL(ref_tracker_free);
344
345#ifdef CONFIG_DEBUG_FS
346#include <linux/debugfs.h>
347
348static struct dentry *ref_tracker_debug_dir = (struct dentry *)-ENOENT;
349
350static void __ostream_printf pr_ostream_seq(struct ostream *stream, char *fmt, ...)
351{
352	va_list args;
353
354	va_start(args, fmt);
355	seq_vprintf(stream->seq, fmt, args);
356	va_end(args);
357}
358
359static int ref_tracker_dir_seq_print(struct ref_tracker_dir *dir, struct seq_file *seq)
360{
361	struct ostream os = { .func = pr_ostream_seq,
362			      .prefix = "",
363			      .seq = seq };
364
365	__ref_tracker_dir_pr_ostream(dir, 16, &os);
366
367	return os.used;
368}
369
370static int ref_tracker_debugfs_show(struct seq_file *f, void *v)
371{
372	struct ref_tracker_dir *dir = f->private;
373	unsigned long index = (unsigned long)dir;
374	unsigned long flags;
375	int ret;
376
377	/*
378	 * "dir" may not exist at this point if ref_tracker_dir_exit() has
379	 * already been called. Take care not to dereference it until its
380	 * legitimacy is established.
381	 *
382	 * The xa_lock is necessary to ensure that "dir" doesn't disappear
383	 * before its lock can be taken. If it's in the hash and not marked
384	 * dead, then it's safe to take dir->lock which prevents
385	 * ref_tracker_dir_exit() from completing. Once the dir->lock is
386	 * acquired, the xa_lock can be released. All of this must be IRQ-safe.
387	 */
388	xa_lock_irqsave(&debugfs_dentries, flags);
389	if (!xa_load(&debugfs_dentries, index) ||
390	    xa_get_mark(&debugfs_dentries, index, REF_TRACKER_DIR_DEAD)) {
391		xa_unlock_irqrestore(&debugfs_dentries, flags);
392		return -ENODATA;
393	}
394
395	spin_lock(&dir->lock);
396	xa_unlock(&debugfs_dentries);
397	ret = ref_tracker_dir_seq_print(dir, f);
398	spin_unlock_irqrestore(&dir->lock, flags);
399	return ret;
400}
401
402static int ref_tracker_debugfs_open(struct inode *inode, struct file *filp)
403{
404	struct ref_tracker_dir *dir = inode->i_private;
405
406	return single_open(filp, ref_tracker_debugfs_show, dir);
407}
408
409static const struct file_operations ref_tracker_debugfs_fops = {
410	.owner		= THIS_MODULE,
411	.open		= ref_tracker_debugfs_open,
412	.read		= seq_read,
413	.llseek		= seq_lseek,
414	.release	= single_release,
415};
416
417/**
418 * ref_tracker_dir_debugfs - create debugfs file for ref_tracker_dir
419 * @dir: ref_tracker_dir to be associated with debugfs file
420 *
421 * In most cases, a debugfs file will be created automatically for every
422 * ref_tracker_dir. If the object was created before debugfs is brought up
423 * then that may fail. In those cases, it is safe to call this at a later
424 * time to create the file.
425 */
426void ref_tracker_dir_debugfs(struct ref_tracker_dir *dir)
427{
428	char name[NAME_MAX + 1];
429	struct dentry *dentry;
430	int ret;
431
432	/* No-op if already created */
433	dentry = xa_load(&debugfs_dentries, (unsigned long)dir);
434	if (dentry && !xa_is_err(dentry))
435		return;
436
437	ret = snprintf(name, sizeof(name), "%s@%p", dir->class, dir);
438	name[sizeof(name) - 1] = '\0';
439
440	if (ret < sizeof(name)) {
441		dentry = debugfs_create_file(name, S_IFREG | 0400,
442					     ref_tracker_debug_dir, dir,
443					     &ref_tracker_debugfs_fops);
444		if (!IS_ERR(dentry)) {
445			void *old;
446
447			old = xa_store_irq(&debugfs_dentries, (unsigned long)dir,
448					   dentry, GFP_KERNEL);
449
450			if (xa_is_err(old))
451				debugfs_remove(dentry);
452			else
453				WARN_ON_ONCE(old);
454		}
455	}
456}
457EXPORT_SYMBOL(ref_tracker_dir_debugfs);
458
459void __ostream_printf ref_tracker_dir_symlink(struct ref_tracker_dir *dir, const char *fmt, ...)
460{
461	char name[NAME_MAX + 1];
462	struct dentry *symlink, *dentry;
463	va_list args;
464	int ret;
465
466	symlink = xa_load(&debugfs_symlinks, (unsigned long)dir);
467	dentry = xa_load(&debugfs_dentries, (unsigned long)dir);
468
469	/* Already created?*/
470	if (symlink && !xa_is_err(symlink))
471		return;
472
473	if (!dentry || xa_is_err(dentry))
474		return;
475
476	va_start(args, fmt);
477	ret = vsnprintf(name, sizeof(name), fmt, args);
478	va_end(args);
479	name[sizeof(name) - 1] = '\0';
480
481	if (ret < sizeof(name)) {
482		symlink = debugfs_create_symlink(name, ref_tracker_debug_dir,
483						 dentry->d_name.name);
484		if (!IS_ERR(symlink)) {
485			void *old;
486
487			old = xa_store_irq(&debugfs_symlinks, (unsigned long)dir,
488					   symlink, GFP_KERNEL);
489			if (xa_is_err(old))
490				debugfs_remove(symlink);
491			else
492				WARN_ON_ONCE(old);
493		}
494	}
495}
496EXPORT_SYMBOL(ref_tracker_dir_symlink);
497
498static void debugfs_reap_work(struct work_struct *work)
499{
500	struct dentry *dentry;
501	unsigned long index;
502	bool reaped;
503
504	do {
505		reaped = false;
506		xa_for_each_marked(&debugfs_symlinks, index, dentry, REF_TRACKER_DIR_DEAD) {
507			xa_erase_irq(&debugfs_symlinks, index);
508			debugfs_remove(dentry);
509			reaped = true;
510		}
511		xa_for_each_marked(&debugfs_dentries, index, dentry, REF_TRACKER_DIR_DEAD) {
512			xa_erase_irq(&debugfs_dentries, index);
513			debugfs_remove(dentry);
514			reaped = true;
515		}
516	} while (reaped);
517}
518
519static int __init ref_tracker_debugfs_postcore_init(void)
520{
521	INIT_WORK(&debugfs_reap_worker, debugfs_reap_work);
522	xa_init_flags(&debugfs_dentries, XA_FLAGS_LOCK_IRQ);
523	xa_init_flags(&debugfs_symlinks, XA_FLAGS_LOCK_IRQ);
524	return 0;
525}
526postcore_initcall(ref_tracker_debugfs_postcore_init);
527
528static int __init ref_tracker_debugfs_late_init(void)
529{
530	ref_tracker_debug_dir = debugfs_create_dir("ref_tracker", NULL);
531	return 0;
532}
533late_initcall(ref_tracker_debugfs_late_init);
534#endif /* CONFIG_DEBUG_FS */