reclaim.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * DAMON-based page reclamation
  4 *
  5 * Author: SeongJae Park <sj@kernel.org>
  6 */
  7
  8#define pr_fmt(fmt) "damon-reclaim: " fmt
  9
 10#include <linux/damon.h>
 11#include <linux/kstrtox.h>
 12#include <linux/module.h>
 13
 14#include "modules-common.h"
 15
 16#ifdef MODULE_PARAM_PREFIX
 17#undef MODULE_PARAM_PREFIX
 18#endif
 19#define MODULE_PARAM_PREFIX "damon_reclaim."
 20
 21/*
 22 * Enable or disable DAMON_RECLAIM.
 23 *
 24 * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``.
 25 * Setting it as ``N`` disables DAMON_RECLAIM.  Note that DAMON_RECLAIM could
 26 * do no real monitoring and reclamation due to the watermarks-based activation
 27 * condition.  Refer to below descriptions for the watermarks parameter for
 28 * this.
 29 */
 30static bool enabled __read_mostly;
 31
 32/*
 33 * Make DAMON_RECLAIM reads the input parameters again, except ``enabled``.
 34 *
 35 * Input parameters that updated while DAMON_RECLAIM is running are not applied
 36 * by default.  Once this parameter is set as ``Y``, DAMON_RECLAIM reads values
 37 * of parametrs except ``enabled`` again.  Once the re-reading is done, this
 38 * parameter is set as ``N``.  If invalid parameters are found while the
 39 * re-reading, DAMON_RECLAIM will be disabled.
 40 */
 41static bool commit_inputs __read_mostly;
 42module_param(commit_inputs, bool, 0600);
 43
 44/*
 45 * Time threshold for cold memory regions identification in microseconds.
 46 *
 47 * If a memory region is not accessed for this or longer time, DAMON_RECLAIM
 48 * identifies the region as cold, and reclaims.  120 seconds by default.
 49 */
 50static unsigned long min_age __read_mostly = 120000000;
 51module_param(min_age, ulong, 0600);
 52
 53static struct damos_quota damon_reclaim_quota = {
 54	/* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */
 55	.ms = 10,
 56	.sz = 128 * 1024 * 1024,
 57	.reset_interval = 1000,
 58	/* Within the quota, page out older regions first. */
 59	.weight_sz = 0,
 60	.weight_nr_accesses = 0,
 61	.weight_age = 1
 62};
 63DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota);
 64
 65/*
 66 * Desired level of memory pressure-stall time in microseconds.
 67 *
 68 * While keeping the caps that set by other quotas, DAMON_RECLAIM automatically
 69 * increases and decreases the effective level of the quota aiming this level of
 70 * memory pressure is incurred.  System-wide ``some`` memory PSI in microseconds
 71 * per quota reset interval (``quota_reset_interval_ms``) is collected and
 72 * compared to this value to see if the aim is satisfied.  Value zero means
 73 * disabling this auto-tuning feature.
 74 *
 75 * Disabled by default.
 76 */
 77static unsigned long quota_mem_pressure_us __read_mostly;
 78module_param(quota_mem_pressure_us, ulong, 0600);
 79
 80/*
 81 * User-specifiable feedback for auto-tuning of the effective quota.
 82 *
 83 * While keeping the caps that set by other quotas, DAMON_RECLAIM automatically
 84 * increases and decreases the effective level of the quota aiming receiving this
 85 * feedback of value ``10,000`` from the user.  DAMON_RECLAIM assumes the feedback
 86 * value and the quota are positively proportional.  Value zero means disabling
 87 * this auto-tuning feature.
 88 *
 89 * Disabled by default.
 90 *
 91 */
 92static unsigned long quota_autotune_feedback __read_mostly;
 93module_param(quota_autotune_feedback, ulong, 0600);
 94
 95static struct damos_watermarks damon_reclaim_wmarks = {
 96	.metric = DAMOS_WMARK_FREE_MEM_RATE,
 97	.interval = 5000000,	/* 5 seconds */
 98	.high = 500,		/* 50 percent */
 99	.mid = 400,		/* 40 percent */
100	.low = 200,		/* 20 percent */
101};
102DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks);
103
104static struct damon_attrs damon_reclaim_mon_attrs = {
105	.sample_interval = 5000,	/* 5 ms */
106	.aggr_interval = 100000,	/* 100 ms */
107	.ops_update_interval = 0,
108	.min_nr_regions = 10,
109	.max_nr_regions = 1000,
110};
111DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs);
112
113/*
114 * Start of the target memory region in physical address.
115 *
116 * The start physical address of memory region that DAMON_RECLAIM will do work
117 * against.  By default, biggest System RAM is used as the region.
118 */
119static unsigned long monitor_region_start __read_mostly;
120module_param(monitor_region_start, ulong, 0600);
121
122/*
123 * End of the target memory region in physical address.
124 *
125 * The end physical address of memory region that DAMON_RECLAIM will do work
126 * against.  By default, biggest System RAM is used as the region.
127 */
128static unsigned long monitor_region_end __read_mostly;
129module_param(monitor_region_end, ulong, 0600);
130
131/*
132 * Scale factor for DAMON_RECLAIM to ops address conversion.
133 *
134 * This parameter must not be set to 0.
135 */
136static unsigned long addr_unit __read_mostly = 1;
137
138/*
139 * Skip anonymous pages reclamation.
140 *
141 * If this parameter is set as ``Y``, DAMON_RECLAIM does not reclaim anonymous
142 * pages.  By default, ``N``.
143 */
144static bool skip_anon __read_mostly;
145module_param(skip_anon, bool, 0600);
146
147/*
148 * PID of the DAMON thread
149 *
150 * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread.
151 * Else, -1.
152 */
153static int kdamond_pid __read_mostly = -1;
154module_param(kdamond_pid, int, 0400);
155
156static struct damos_stat damon_reclaim_stat;
157DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat,
158		reclaim_tried_regions, reclaimed_regions, quota_exceeds);
159
160static struct damon_ctx *ctx;
161static struct damon_target *target;
162
163static struct damos *damon_reclaim_new_scheme(void)
164{
165	struct damos_access_pattern pattern = {
166		/* Find regions having PAGE_SIZE or larger size */
167		.min_sz_region = PAGE_SIZE,
168		.max_sz_region = ULONG_MAX,
169		/* and not accessed at all */
170		.min_nr_accesses = 0,
171		.max_nr_accesses = 0,
172		/* for min_age or more micro-seconds */
173		.min_age_region = min_age /
174			damon_reclaim_mon_attrs.aggr_interval,
175		.max_age_region = UINT_MAX,
176	};
177
178	return damon_new_scheme(
179			&pattern,
180			/* page out those, as soon as found */
181			DAMOS_PAGEOUT,
182			/* for each aggregation interval */
183			0,
184			/* under the quota. */
185			&damon_reclaim_quota,
186			/* (De)activate this according to the watermarks. */
187			&damon_reclaim_wmarks,
188			NUMA_NO_NODE);
189}
190
191static int damon_reclaim_apply_parameters(void)
192{
193	struct damon_ctx *param_ctx;
194	struct damon_target *param_target;
195	struct damos *scheme;
196	struct damos_quota_goal *goal;
197	struct damos_filter *filter;
198	int err;
199
200	err = damon_modules_new_paddr_ctx_target(&param_ctx, &param_target);
201	if (err)
202		return err;
203
204	/*
205	 * If monitor_region_start/end are unset, always silently
206	 * reset addr_unit to 1.
207	 */
208	if (!monitor_region_start && !monitor_region_end)
209		addr_unit = 1;
210	param_ctx->addr_unit = addr_unit;
211	param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1);
212
213	if (!damon_reclaim_mon_attrs.aggr_interval) {
214		err = -EINVAL;
215		goto out;
216	}
217
218	err = damon_set_attrs(param_ctx, &damon_reclaim_mon_attrs);
219	if (err)
220		goto out;
221
222	err = -ENOMEM;
223	scheme = damon_reclaim_new_scheme();
224	if (!scheme)
225		goto out;
226	damon_set_schemes(param_ctx, &scheme, 1);
227
228	if (quota_mem_pressure_us) {
229		goal = damos_new_quota_goal(DAMOS_QUOTA_SOME_MEM_PSI_US,
230				quota_mem_pressure_us);
231		if (!goal)
232			goto out;
233		damos_add_quota_goal(&scheme->quota, goal);
234	}
235
236	if (quota_autotune_feedback) {
237		goal = damos_new_quota_goal(DAMOS_QUOTA_USER_INPUT, 10000);
238		if (!goal)
239			goto out;
240		goal->current_value = quota_autotune_feedback;
241		damos_add_quota_goal(&scheme->quota, goal);
242	}
243
244	if (skip_anon) {
245		filter = damos_new_filter(DAMOS_FILTER_TYPE_ANON, true, false);
246		if (!filter)
247			goto out;
248		damos_add_filter(scheme, filter);
249	}
250
251	err = damon_set_region_biggest_system_ram_default(param_target,
252					&monitor_region_start,
253					&monitor_region_end,
254					param_ctx->min_region_sz);
255	if (err)
256		goto out;
257	err = damon_commit_ctx(ctx, param_ctx);
258out:
259	damon_destroy_ctx(param_ctx);
260	return err;
261}
262
263static int damon_reclaim_handle_commit_inputs(void)
264{
265	int err;
266
267	if (!commit_inputs)
268		return 0;
269
270	err = damon_reclaim_apply_parameters();
271	commit_inputs = false;
272	return err;
273}
274
275static int damon_reclaim_damon_call_fn(void *arg)
276{
277	struct damon_ctx *c = arg;
278	struct damos *s;
279
280	/* update the stats parameter */
281	damon_for_each_scheme(s, c)
282		damon_reclaim_stat = s->stat;
283
284	return damon_reclaim_handle_commit_inputs();
285}
286
287static struct damon_call_control call_control = {
288	.fn = damon_reclaim_damon_call_fn,
289	.repeat = true,
290};
291
292static int damon_reclaim_turn(bool on)
293{
294	int err;
295
296	if (!on) {
297		err = damon_stop(&ctx, 1);
298		if (!err)
299			kdamond_pid = -1;
300		return err;
301	}
302
303	err = damon_reclaim_apply_parameters();
304	if (err)
305		return err;
306
307	err = damon_start(&ctx, 1, true);
308	if (err)
309		return err;
310	kdamond_pid = ctx->kdamond->pid;
311	return damon_call(ctx, &call_control);
312}
313
314static int damon_reclaim_addr_unit_store(const char *val,
315		const struct kernel_param *kp)
316{
317	unsigned long input_addr_unit;
318	int err = kstrtoul(val, 0, &input_addr_unit);
319
320	if (err)
321		return err;
322	if (!input_addr_unit)
323		return -EINVAL;
324
325	addr_unit = input_addr_unit;
326	return 0;
327}
328
329static const struct kernel_param_ops addr_unit_param_ops = {
330	.set = damon_reclaim_addr_unit_store,
331	.get = param_get_ulong,
332};
333
334module_param_cb(addr_unit, &addr_unit_param_ops, &addr_unit, 0600);
335MODULE_PARM_DESC(addr_unit,
336	"Scale factor for DAMON_RECLAIM to ops address conversion (default: 1)");
337
338static int damon_reclaim_enabled_store(const char *val,
339		const struct kernel_param *kp)
340{
341	bool is_enabled = enabled;
342	bool enable;
343	int err;
344
345	err = kstrtobool(val, &enable);
346	if (err)
347		return err;
348
349	if (is_enabled == enable)
350		return 0;
351
352	/* Called before init function.  The function will handle this. */
353	if (!damon_initialized())
354		goto set_param_out;
355
356	err = damon_reclaim_turn(enable);
357	if (err)
358		return err;
359
360set_param_out:
361	enabled = enable;
362	return err;
363}
364
365static const struct kernel_param_ops enabled_param_ops = {
366	.set = damon_reclaim_enabled_store,
367	.get = param_get_bool,
368};
369
370module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
371MODULE_PARM_DESC(enabled,
372	"Enable or disable DAMON_RECLAIM (default: disabled)");
373
374static int __init damon_reclaim_init(void)
375{
376	int err;
377
378	if (!damon_initialized()) {
379		err = -ENOMEM;
380		goto out;
381	}
382	err = damon_modules_new_paddr_ctx_target(&ctx, &target);
383	if (err)
384		goto out;
385
386	call_control.data = ctx;
387
388	/* 'enabled' has set before this function, probably via command line */
389	if (enabled)
390		err = damon_reclaim_turn(true);
391
392out:
393	if (err && enabled)
394		enabled = false;
395	return err;
396}
397
398module_init(damon_reclaim_init);