mtier.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * memory tiering: migrate cold pages in node 0 and hot pages in node 1 to node
  4 * 1 and node 0, respectively.  Adjust the hotness/coldness threshold aiming
  5 * resulting 99.6 % node 0 utilization ratio.
  6 */
  7
  8#define pr_fmt(fmt) "damon_sample_mtier: " fmt
  9
 10#include <linux/damon.h>
 11#include <linux/init.h>
 12#include <linux/kernel.h>
 13#include <linux/module.h>
 14
 15#ifdef MODULE_PARAM_PREFIX
 16#undef MODULE_PARAM_PREFIX
 17#endif
 18#define MODULE_PARAM_PREFIX "damon_sample_mtier."
 19
 20static unsigned long node0_start_addr __read_mostly;
 21module_param(node0_start_addr, ulong, 0600);
 22
 23static unsigned long node0_end_addr __read_mostly;
 24module_param(node0_end_addr, ulong, 0600);
 25
 26static unsigned long node1_start_addr __read_mostly;
 27module_param(node1_start_addr, ulong, 0600);
 28
 29static unsigned long node1_end_addr __read_mostly;
 30module_param(node1_end_addr, ulong, 0600);
 31
 32static unsigned long node0_mem_used_bp __read_mostly = 9970;
 33module_param(node0_mem_used_bp, ulong, 0600);
 34
 35static unsigned long node0_mem_free_bp __read_mostly = 50;
 36module_param(node0_mem_free_bp, ulong, 0600);
 37
 38static int damon_sample_mtier_enable_store(
 39		const char *val, const struct kernel_param *kp);
 40
 41static const struct kernel_param_ops enabled_param_ops = {
 42	.set = damon_sample_mtier_enable_store,
 43	.get = param_get_bool,
 44};
 45
 46static bool enabled __read_mostly;
 47module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
 48MODULE_PARM_DESC(enabled, "Enable or disable DAMON_SAMPLE_MTIER");
 49
 50static bool detect_node_addresses __read_mostly;
 51module_param(detect_node_addresses, bool, 0600);
 52
 53static struct damon_ctx *ctxs[2];
 54
 55struct region_range {
 56	phys_addr_t start;
 57	phys_addr_t end;
 58};
 59
 60static int nid_to_phys(int target_node, struct region_range *range)
 61{
 62	if (!node_online(target_node)) {
 63		pr_err("NUMA node %d is not online\n", target_node);
 64		return -EINVAL;
 65	}
 66
 67	range->start = PFN_PHYS(node_start_pfn(target_node));
 68	range->end  = PFN_PHYS(node_end_pfn(target_node));
 69
 70	return 0;
 71}
 72
 73static struct damon_ctx *damon_sample_mtier_build_ctx(bool promote)
 74{
 75	struct damon_ctx *ctx;
 76	struct damon_attrs attrs;
 77	struct damon_target *target;
 78	struct damon_region *region;
 79	struct damos *scheme;
 80	struct damos_quota_goal *quota_goal;
 81	struct damos_filter *filter;
 82	struct region_range addr;
 83	int ret;
 84
 85	ctx = damon_new_ctx();
 86	if (!ctx)
 87		return NULL;
 88	attrs = (struct damon_attrs) {
 89		.sample_interval = 5 * USEC_PER_MSEC,
 90		.aggr_interval = 100 * USEC_PER_MSEC,
 91		.ops_update_interval = 60 * USEC_PER_MSEC * MSEC_PER_SEC,
 92		.min_nr_regions = 10,
 93		.max_nr_regions = 1000,
 94	};
 95
 96	/*
 97	 * auto-tune sampling and aggregation interval aiming 4% DAMON-observed
 98	 * accesses ratio, keeping sampling interval in [5ms, 10s] range.
 99	 */
100	attrs.intervals_goal = (struct damon_intervals_goal) {
101		.access_bp = 400, .aggrs = 3,
102		.min_sample_us = 5000, .max_sample_us = 10000000,
103	};
104	if (damon_set_attrs(ctx, &attrs))
105		goto free_out;
106	if (damon_select_ops(ctx, DAMON_OPS_PADDR))
107		goto free_out;
108
109	target = damon_new_target();
110	if (!target)
111		goto free_out;
112	damon_add_target(ctx, target);
113
114	if (detect_node_addresses) {
115		ret = promote ? nid_to_phys(1, &addr) : nid_to_phys(0, &addr);
116		if (ret)
117			goto free_out;
118	} else {
119		addr.start = promote ? node1_start_addr : node0_start_addr;
120		addr.end = promote ? node1_end_addr : node0_end_addr;
121	}
122
123	region = damon_new_region(addr.start, addr.end);
124	if (!region)
125		goto free_out;
126	damon_add_region(region, target);
127
128	scheme = damon_new_scheme(
129			/* access pattern */
130			&(struct damos_access_pattern) {
131				.min_sz_region = PAGE_SIZE,
132				.max_sz_region = ULONG_MAX,
133				.min_nr_accesses = promote ? 1 : 0,
134				.max_nr_accesses = promote ? UINT_MAX : 0,
135				.min_age_region = 0,
136				.max_age_region = UINT_MAX},
137			/* action */
138			promote ? DAMOS_MIGRATE_HOT : DAMOS_MIGRATE_COLD,
139			1000000,	/* apply interval (1s) */
140			&(struct damos_quota){
141				/* 200 MiB per sec by most */
142				.reset_interval = 1000,
143				.sz = 200 * 1024 * 1024,
144				/* ignore size of region when prioritizing */
145				.weight_sz = 0,
146				.weight_nr_accesses = 100,
147				.weight_age = 100,
148			},
149			&(struct damos_watermarks){},
150			promote ? 0 : 1);	/* migrate target node id */
151	if (!scheme)
152		goto free_out;
153	damon_set_schemes(ctx, &scheme, 1);
154	quota_goal = damos_new_quota_goal(
155			promote ? DAMOS_QUOTA_NODE_MEM_USED_BP :
156			DAMOS_QUOTA_NODE_MEM_FREE_BP,
157			promote ? node0_mem_used_bp : node0_mem_free_bp);
158	if (!quota_goal)
159		goto free_out;
160	quota_goal->nid = 0;
161	damos_add_quota_goal(&scheme->quota, quota_goal);
162	filter = damos_new_filter(DAMOS_FILTER_TYPE_YOUNG, true, promote);
163	if (!filter)
164		goto free_out;
165	damos_add_filter(scheme, filter);
166	return ctx;
167free_out:
168	damon_destroy_ctx(ctx);
169	return NULL;
170}
171
172static int damon_sample_mtier_start(void)
173{
174	struct damon_ctx *ctx;
175
176	ctx = damon_sample_mtier_build_ctx(true);
177	if (!ctx)
178		return -ENOMEM;
179	ctxs[0] = ctx;
180	ctx = damon_sample_mtier_build_ctx(false);
181	if (!ctx) {
182		damon_destroy_ctx(ctxs[0]);
183		return -ENOMEM;
184	}
185	ctxs[1] = ctx;
186	return damon_start(ctxs, 2, true);
187}
188
189static void damon_sample_mtier_stop(void)
190{
191	damon_stop(ctxs, 2);
192	damon_destroy_ctx(ctxs[0]);
193	damon_destroy_ctx(ctxs[1]);
194}
195
196static int damon_sample_mtier_enable_store(
197		const char *val, const struct kernel_param *kp)
198{
199	bool is_enabled = enabled;
200	int err;
201
202	err = kstrtobool(val, &enabled);
203	if (err)
204		return err;
205
206	if (enabled == is_enabled)
207		return 0;
208
209	if (!damon_initialized())
210		return 0;
211
212	if (enabled) {
213		err = damon_sample_mtier_start();
214		if (err)
215			enabled = false;
216		return err;
217	}
218	damon_sample_mtier_stop();
219	return 0;
220}
221
222static int __init damon_sample_mtier_init(void)
223{
224	int err = 0;
225
226	if (!damon_initialized()) {
227		if (enabled)
228			enabled = false;
229		return -ENOMEM;
230	}
231
232	if (enabled) {
233		err = damon_sample_mtier_start();
234		if (err)
235			enabled = false;
236	}
237	return 0;
238}
239
240module_init(damon_sample_mtier_init);