stat.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Shows data access monitoring resutls in simple metrics.
  4 */
  5
  6#define pr_fmt(fmt) "damon-stat: " fmt
  7
  8#include <linux/damon.h>
  9#include <linux/init.h>
 10#include <linux/kernel.h>
 11#include <linux/module.h>
 12#include <linux/sort.h>
 13
 14#ifdef MODULE_PARAM_PREFIX
 15#undef MODULE_PARAM_PREFIX
 16#endif
 17#define MODULE_PARAM_PREFIX "damon_stat."
 18
 19static int damon_stat_enabled_store(
 20		const char *val, const struct kernel_param *kp);
 21
 22static const struct kernel_param_ops enabled_param_ops = {
 23	.set = damon_stat_enabled_store,
 24	.get = param_get_bool,
 25};
 26
 27static bool enabled __read_mostly = IS_ENABLED(
 28	CONFIG_DAMON_STAT_ENABLED_DEFAULT);
 29module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
 30MODULE_PARM_DESC(enabled, "Enable of disable DAMON_STAT");
 31
 32static unsigned long estimated_memory_bandwidth __read_mostly;
 33module_param(estimated_memory_bandwidth, ulong, 0400);
 34MODULE_PARM_DESC(estimated_memory_bandwidth,
 35		"Estimated memory bandwidth usage in bytes per second");
 36
 37static long memory_idle_ms_percentiles[101] __read_mostly = {0,};
 38module_param_array(memory_idle_ms_percentiles, long, NULL, 0400);
 39MODULE_PARM_DESC(memory_idle_ms_percentiles,
 40		"Memory idle time percentiles in milliseconds");
 41
 42static unsigned long aggr_interval_us;
 43module_param(aggr_interval_us, ulong, 0400);
 44MODULE_PARM_DESC(aggr_interval_us,
 45		"Current tuned aggregation interval in microseconds");
 46
 47static struct damon_ctx *damon_stat_context;
 48
 49static unsigned long damon_stat_last_refresh_jiffies;
 50
 51static void damon_stat_set_estimated_memory_bandwidth(struct damon_ctx *c)
 52{
 53	struct damon_target *t;
 54	struct damon_region *r;
 55	unsigned long access_bytes = 0;
 56
 57	damon_for_each_target(t, c) {
 58		damon_for_each_region(r, t)
 59			access_bytes += (r->ar.end - r->ar.start) *
 60				r->nr_accesses;
 61	}
 62	estimated_memory_bandwidth = access_bytes * USEC_PER_MSEC *
 63		MSEC_PER_SEC / c->attrs.aggr_interval;
 64}
 65
 66static int damon_stat_idletime(const struct damon_region *r)
 67{
 68	if (r->nr_accesses)
 69		return -1 * (r->age + 1);
 70	return r->age + 1;
 71}
 72
 73static int damon_stat_cmp_regions(const void *a, const void *b)
 74{
 75	const struct damon_region *ra = *(const struct damon_region **)a;
 76	const struct damon_region *rb = *(const struct damon_region **)b;
 77
 78	return damon_stat_idletime(ra) - damon_stat_idletime(rb);
 79}
 80
 81static int damon_stat_sort_regions(struct damon_ctx *c,
 82		struct damon_region ***sorted_ptr, int *nr_regions_ptr,
 83		unsigned long *total_sz_ptr)
 84{
 85	struct damon_target *t;
 86	struct damon_region *r;
 87	struct damon_region **region_pointers;
 88	unsigned int nr_regions = 0;
 89	unsigned long total_sz = 0;
 90
 91	damon_for_each_target(t, c) {
 92		/* there is only one target */
 93		region_pointers = kmalloc_array(damon_nr_regions(t),
 94				sizeof(*region_pointers), GFP_KERNEL);
 95		if (!region_pointers)
 96			return -ENOMEM;
 97		damon_for_each_region(r, t) {
 98			region_pointers[nr_regions++] = r;
 99			total_sz += r->ar.end - r->ar.start;
100		}
101	}
102	sort(region_pointers, nr_regions, sizeof(*region_pointers),
103			damon_stat_cmp_regions, NULL);
104	*sorted_ptr = region_pointers;
105	*nr_regions_ptr = nr_regions;
106	*total_sz_ptr = total_sz;
107	return 0;
108}
109
110static void damon_stat_set_idletime_percentiles(struct damon_ctx *c)
111{
112	struct damon_region **sorted_regions, *region;
113	int nr_regions;
114	unsigned long total_sz, accounted_bytes = 0;
115	int err, i, next_percentile = 0;
116
117	err = damon_stat_sort_regions(c, &sorted_regions, &nr_regions,
118			&total_sz);
119	if (err)
120		return;
121	for (i = 0; i < nr_regions; i++) {
122		region = sorted_regions[i];
123		accounted_bytes += region->ar.end - region->ar.start;
124		while (next_percentile <= accounted_bytes * 100 / total_sz)
125			memory_idle_ms_percentiles[next_percentile++] =
126				damon_stat_idletime(region) *
127				(long)c->attrs.aggr_interval / USEC_PER_MSEC;
128	}
129	kfree(sorted_regions);
130}
131
132static int damon_stat_damon_call_fn(void *data)
133{
134	struct damon_ctx *c = data;
135
136	/* avoid unnecessarily frequent stat update */
137	if (time_before_eq(jiffies, damon_stat_last_refresh_jiffies +
138				msecs_to_jiffies(5 * MSEC_PER_SEC)))
139		return 0;
140	damon_stat_last_refresh_jiffies = jiffies;
141
142	aggr_interval_us = c->attrs.aggr_interval;
143	damon_stat_set_estimated_memory_bandwidth(c);
144	damon_stat_set_idletime_percentiles(c);
145	return 0;
146}
147
148struct damon_stat_system_ram_range_walk_arg {
149	bool walked;
150	struct resource res;
151};
152
153static int damon_stat_system_ram_walk_fn(struct resource *res, void *arg)
154{
155	struct damon_stat_system_ram_range_walk_arg *a = arg;
156
157	if (!a->walked) {
158		a->walked = true;
159		a->res.start = res->start;
160	}
161	a->res.end = res->end;
162	return 0;
163}
164
165static unsigned long damon_stat_res_to_core_addr(resource_size_t ra,
166		unsigned long addr_unit)
167{
168	/*
169	 * Use div_u64() for avoiding linking errors related with __udivdi3,
170	 * __aeabi_uldivmod, or similar problems.  This should also improve the
171	 * performance optimization (read div_u64() comment for the detail).
172	 */
173	if (sizeof(ra) == 8 && sizeof(addr_unit) == 4)
174		return div_u64(ra, addr_unit);
175	return ra / addr_unit;
176}
177
178static int damon_stat_set_monitoring_region(struct damon_target *t,
179		unsigned long addr_unit, unsigned long min_region_sz)
180{
181	struct damon_addr_range addr_range;
182	struct damon_stat_system_ram_range_walk_arg arg = {};
183
184	walk_system_ram_res(0, -1, &arg, damon_stat_system_ram_walk_fn);
185	if (!arg.walked)
186		return -EINVAL;
187	addr_range.start = damon_stat_res_to_core_addr(
188			arg.res.start, addr_unit);
189	addr_range.end = damon_stat_res_to_core_addr(
190			arg.res.end + 1, addr_unit);
191	if (addr_range.end <= addr_range.start)
192		return -EINVAL;
193	return damon_set_regions(t, &addr_range, 1, min_region_sz);
194}
195
196static struct damon_ctx *damon_stat_build_ctx(void)
197{
198	struct damon_ctx *ctx;
199	struct damon_attrs attrs;
200	struct damon_target *target;
201
202	ctx = damon_new_ctx();
203	if (!ctx)
204		return NULL;
205	attrs = (struct damon_attrs) {
206		.sample_interval = 5 * USEC_PER_MSEC,
207		.aggr_interval = 100 * USEC_PER_MSEC,
208		.ops_update_interval = 60 * USEC_PER_MSEC * MSEC_PER_SEC,
209		.min_nr_regions = 10,
210		.max_nr_regions = 1000,
211	};
212	/*
213	 * auto-tune sampling and aggregation interval aiming 4% DAMON-observed
214	 * accesses ratio, keeping sampling interval in [5ms, 10s] range.
215	 */
216	attrs.intervals_goal = (struct damon_intervals_goal) {
217		.access_bp = 400, .aggrs = 3,
218		.min_sample_us = 5000, .max_sample_us = 10000000,
219	};
220	if (damon_set_attrs(ctx, &attrs))
221		goto free_out;
222
223	/*
224	 * auto-tune sampling and aggregation interval aiming 4% DAMON-observed
225	 * accesses ratio, keeping sampling interval in [5ms, 10s] range.
226	 */
227	ctx->attrs.intervals_goal = (struct damon_intervals_goal) {
228		.access_bp = 400, .aggrs = 3,
229		.min_sample_us = 5000, .max_sample_us = 10000000,
230	};
231	if (damon_select_ops(ctx, DAMON_OPS_PADDR))
232		goto free_out;
233
234	target = damon_new_target();
235	if (!target)
236		goto free_out;
237	damon_add_target(ctx, target);
238	if (damon_stat_set_monitoring_region(target, ctx->addr_unit,
239				ctx->min_region_sz))
240		goto free_out;
241	return ctx;
242free_out:
243	damon_destroy_ctx(ctx);
244	return NULL;
245}
246
247static struct damon_call_control call_control = {
248	.fn = damon_stat_damon_call_fn,
249	.repeat = true,
250};
251
252static int damon_stat_start(void)
253{
254	int err;
255
256	if (damon_stat_context) {
257		if (damon_is_running(damon_stat_context))
258			return -EAGAIN;
259		damon_destroy_ctx(damon_stat_context);
260	}
261
262	damon_stat_context = damon_stat_build_ctx();
263	if (!damon_stat_context)
264		return -ENOMEM;
265	err = damon_start(&damon_stat_context, 1, true);
266	if (err)
267		return err;
268
269	damon_stat_last_refresh_jiffies = jiffies;
270	call_control.data = damon_stat_context;
271	return damon_call(damon_stat_context, &call_control);
272}
273
274static void damon_stat_stop(void)
275{
276	damon_stop(&damon_stat_context, 1);
277	damon_destroy_ctx(damon_stat_context);
278	damon_stat_context = NULL;
279}
280
281static int damon_stat_enabled_store(
282		const char *val, const struct kernel_param *kp)
283{
284	bool is_enabled = enabled;
285	int err;
286
287	err = kstrtobool(val, &enabled);
288	if (err)
289		return err;
290
291	if (is_enabled == enabled)
292		return 0;
293
294	if (!damon_initialized())
295		/*
296		 * probably called from command line parsing (parse_args()).
297		 * Cannot call damon_new_ctx().  Let damon_stat_init() handle.
298		 */
299		return 0;
300
301	if (enabled) {
302		err = damon_stat_start();
303		if (err)
304			enabled = false;
305		return err;
306	}
307	damon_stat_stop();
308	return 0;
309}
310
311static int __init damon_stat_init(void)
312{
313	int err = 0;
314
315	if (!damon_initialized()) {
316		err = -ENOMEM;
317		goto out;
318	}
319
320	/* probably set via command line */
321	if (enabled)
322		err = damon_stat_start();
323
324out:
325	if (err && enabled)
326		enabled = false;
327	return err;
328}
329
330module_init(damon_stat_init);