core.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Data Access Monitor
   4 *
   5 * Author: SeongJae Park <sj@kernel.org>
   6 */
   7
   8#define pr_fmt(fmt) "damon: " fmt
   9
  10#include <linux/damon.h>
  11#include <linux/delay.h>
  12#include <linux/kthread.h>
  13#include <linux/memcontrol.h>
  14#include <linux/mm.h>
  15#include <linux/psi.h>
  16#include <linux/slab.h>
  17#include <linux/string.h>
  18#include <linux/string_choices.h>
  19
  20#define CREATE_TRACE_POINTS
  21#include <trace/events/damon.h>
  22
  23static DEFINE_MUTEX(damon_lock);
  24static int nr_running_ctxs;
  25static bool running_exclusive_ctxs;
  26
  27static DEFINE_MUTEX(damon_ops_lock);
  28static struct damon_operations damon_registered_ops[NR_DAMON_OPS];
  29
  30static struct kmem_cache *damon_region_cache __ro_after_init;
  31
  32/* Should be called under damon_ops_lock with id smaller than NR_DAMON_OPS */
  33static bool __damon_is_registered_ops(enum damon_ops_id id)
  34{
  35	struct damon_operations empty_ops = {};
  36
  37	if (!memcmp(&empty_ops, &damon_registered_ops[id], sizeof(empty_ops)))
  38		return false;
  39	return true;
  40}
  41
  42/**
  43 * damon_is_registered_ops() - Check if a given damon_operations is registered.
  44 * @id:	Id of the damon_operations to check if registered.
  45 *
  46 * Return: true if the ops is set, false otherwise.
  47 */
  48bool damon_is_registered_ops(enum damon_ops_id id)
  49{
  50	bool registered;
  51
  52	if (id >= NR_DAMON_OPS)
  53		return false;
  54	mutex_lock(&damon_ops_lock);
  55	registered = __damon_is_registered_ops(id);
  56	mutex_unlock(&damon_ops_lock);
  57	return registered;
  58}
  59
  60/**
  61 * damon_register_ops() - Register a monitoring operations set to DAMON.
  62 * @ops:	monitoring operations set to register.
  63 *
  64 * This function registers a monitoring operations set of valid &struct
  65 * damon_operations->id so that others can find and use them later.
  66 *
  67 * Return: 0 on success, negative error code otherwise.
  68 */
  69int damon_register_ops(struct damon_operations *ops)
  70{
  71	int err = 0;
  72
  73	if (ops->id >= NR_DAMON_OPS)
  74		return -EINVAL;
  75
  76	mutex_lock(&damon_ops_lock);
  77	/* Fail for already registered ops */
  78	if (__damon_is_registered_ops(ops->id))
  79		err = -EINVAL;
  80	else
  81		damon_registered_ops[ops->id] = *ops;
  82	mutex_unlock(&damon_ops_lock);
  83	return err;
  84}
  85
  86/**
  87 * damon_select_ops() - Select a monitoring operations to use with the context.
  88 * @ctx:	monitoring context to use the operations.
  89 * @id:		id of the registered monitoring operations to select.
  90 *
  91 * This function finds registered monitoring operations set of @id and make
  92 * @ctx to use it.
  93 *
  94 * Return: 0 on success, negative error code otherwise.
  95 */
  96int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id)
  97{
  98	int err = 0;
  99
 100	if (id >= NR_DAMON_OPS)
 101		return -EINVAL;
 102
 103	mutex_lock(&damon_ops_lock);
 104	if (!__damon_is_registered_ops(id))
 105		err = -EINVAL;
 106	else
 107		ctx->ops = damon_registered_ops[id];
 108	mutex_unlock(&damon_ops_lock);
 109	return err;
 110}
 111
 112/*
 113 * Construct a damon_region struct
 114 *
 115 * Returns the pointer to the new struct if success, or NULL otherwise
 116 */
 117struct damon_region *damon_new_region(unsigned long start, unsigned long end)
 118{
 119	struct damon_region *region;
 120
 121	region = kmem_cache_alloc(damon_region_cache, GFP_KERNEL);
 122	if (!region)
 123		return NULL;
 124
 125	region->ar.start = start;
 126	region->ar.end = end;
 127	region->nr_accesses = 0;
 128	region->nr_accesses_bp = 0;
 129	INIT_LIST_HEAD(&region->list);
 130
 131	region->age = 0;
 132	region->last_nr_accesses = 0;
 133
 134	return region;
 135}
 136
 137void damon_add_region(struct damon_region *r, struct damon_target *t)
 138{
 139	list_add_tail(&r->list, &t->regions_list);
 140	t->nr_regions++;
 141}
 142
 143static void damon_del_region(struct damon_region *r, struct damon_target *t)
 144{
 145	list_del(&r->list);
 146	t->nr_regions--;
 147}
 148
 149static void damon_free_region(struct damon_region *r)
 150{
 151	kmem_cache_free(damon_region_cache, r);
 152}
 153
 154void damon_destroy_region(struct damon_region *r, struct damon_target *t)
 155{
 156	damon_del_region(r, t);
 157	damon_free_region(r);
 158}
 159
 160/*
 161 * Check whether a region is intersecting an address range
 162 *
 163 * Returns true if it is.
 164 */
 165static bool damon_intersect(struct damon_region *r,
 166		struct damon_addr_range *re)
 167{
 168	return !(r->ar.end <= re->start || re->end <= r->ar.start);
 169}
 170
 171/*
 172 * Fill holes in regions with new regions.
 173 */
 174static int damon_fill_regions_holes(struct damon_region *first,
 175		struct damon_region *last, struct damon_target *t)
 176{
 177	struct damon_region *r = first;
 178
 179	damon_for_each_region_from(r, t) {
 180		struct damon_region *next, *newr;
 181
 182		if (r == last)
 183			break;
 184		next = damon_next_region(r);
 185		if (r->ar.end != next->ar.start) {
 186			newr = damon_new_region(r->ar.end, next->ar.start);
 187			if (!newr)
 188				return -ENOMEM;
 189			damon_insert_region(newr, r, next, t);
 190		}
 191	}
 192	return 0;
 193}
 194
 195/*
 196 * damon_set_regions() - Set regions of a target for given address ranges.
 197 * @t:		the given target.
 198 * @ranges:	array of new monitoring target ranges.
 199 * @nr_ranges:	length of @ranges.
 200 * @min_region_sz:	minimum region size.
 201 *
 202 * This function adds new regions to, or modify existing regions of a
 203 * monitoring target to fit in specific ranges.
 204 *
 205 * Return: 0 if success, or negative error code otherwise.
 206 */
 207int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges,
 208		unsigned int nr_ranges, unsigned long min_region_sz)
 209{
 210	struct damon_region *r, *next;
 211	unsigned int i;
 212	int err;
 213
 214	/* Remove regions which are not in the new ranges */
 215	damon_for_each_region_safe(r, next, t) {
 216		for (i = 0; i < nr_ranges; i++) {
 217			if (damon_intersect(r, &ranges[i]))
 218				break;
 219		}
 220		if (i == nr_ranges)
 221			damon_destroy_region(r, t);
 222	}
 223
 224	r = damon_first_region(t);
 225	/* Add new regions or resize existing regions to fit in the ranges */
 226	for (i = 0; i < nr_ranges; i++) {
 227		struct damon_region *first = NULL, *last, *newr;
 228		struct damon_addr_range *range;
 229
 230		range = &ranges[i];
 231		/* Get the first/last regions intersecting with the range */
 232		damon_for_each_region_from(r, t) {
 233			if (damon_intersect(r, range)) {
 234				if (!first)
 235					first = r;
 236				last = r;
 237			}
 238			if (r->ar.start >= range->end)
 239				break;
 240		}
 241		if (!first) {
 242			/* no region intersects with this range */
 243			newr = damon_new_region(
 244					ALIGN_DOWN(range->start,
 245						min_region_sz),
 246					ALIGN(range->end, min_region_sz));
 247			if (!newr)
 248				return -ENOMEM;
 249			damon_insert_region(newr, damon_prev_region(r), r, t);
 250		} else {
 251			/* resize intersecting regions to fit in this range */
 252			first->ar.start = ALIGN_DOWN(range->start,
 253					min_region_sz);
 254			last->ar.end = ALIGN(range->end, min_region_sz);
 255
 256			/* fill possible holes in the range */
 257			err = damon_fill_regions_holes(first, last, t);
 258			if (err)
 259				return err;
 260		}
 261	}
 262	return 0;
 263}
 264
 265struct damos_filter *damos_new_filter(enum damos_filter_type type,
 266		bool matching, bool allow)
 267{
 268	struct damos_filter *filter;
 269
 270	filter = kmalloc(sizeof(*filter), GFP_KERNEL);
 271	if (!filter)
 272		return NULL;
 273	filter->type = type;
 274	filter->matching = matching;
 275	filter->allow = allow;
 276	INIT_LIST_HEAD(&filter->list);
 277	return filter;
 278}
 279
 280/**
 281 * damos_filter_for_ops() - Return if the filter is ops-hndled one.
 282 * @type:	type of the filter.
 283 *
 284 * Return: true if the filter of @type needs to be handled by ops layer, false
 285 * otherwise.
 286 */
 287bool damos_filter_for_ops(enum damos_filter_type type)
 288{
 289	switch (type) {
 290	case DAMOS_FILTER_TYPE_ADDR:
 291	case DAMOS_FILTER_TYPE_TARGET:
 292		return false;
 293	default:
 294		break;
 295	}
 296	return true;
 297}
 298
 299void damos_add_filter(struct damos *s, struct damos_filter *f)
 300{
 301	if (damos_filter_for_ops(f->type))
 302		list_add_tail(&f->list, &s->ops_filters);
 303	else
 304		list_add_tail(&f->list, &s->core_filters);
 305}
 306
 307static void damos_del_filter(struct damos_filter *f)
 308{
 309	list_del(&f->list);
 310}
 311
 312static void damos_free_filter(struct damos_filter *f)
 313{
 314	kfree(f);
 315}
 316
 317void damos_destroy_filter(struct damos_filter *f)
 318{
 319	damos_del_filter(f);
 320	damos_free_filter(f);
 321}
 322
 323struct damos_quota_goal *damos_new_quota_goal(
 324		enum damos_quota_goal_metric metric,
 325		unsigned long target_value)
 326{
 327	struct damos_quota_goal *goal;
 328
 329	goal = kmalloc(sizeof(*goal), GFP_KERNEL);
 330	if (!goal)
 331		return NULL;
 332	goal->metric = metric;
 333	goal->target_value = target_value;
 334	INIT_LIST_HEAD(&goal->list);
 335	return goal;
 336}
 337
 338void damos_add_quota_goal(struct damos_quota *q, struct damos_quota_goal *g)
 339{
 340	list_add_tail(&g->list, &q->goals);
 341}
 342
 343static void damos_del_quota_goal(struct damos_quota_goal *g)
 344{
 345	list_del(&g->list);
 346}
 347
 348static void damos_free_quota_goal(struct damos_quota_goal *g)
 349{
 350	kfree(g);
 351}
 352
 353void damos_destroy_quota_goal(struct damos_quota_goal *g)
 354{
 355	damos_del_quota_goal(g);
 356	damos_free_quota_goal(g);
 357}
 358
 359/* initialize fields of @quota that normally API users wouldn't set */
 360static struct damos_quota *damos_quota_init(struct damos_quota *quota)
 361{
 362	quota->esz = 0;
 363	quota->total_charged_sz = 0;
 364	quota->total_charged_ns = 0;
 365	quota->charged_sz = 0;
 366	quota->charged_from = 0;
 367	quota->charge_target_from = NULL;
 368	quota->charge_addr_from = 0;
 369	quota->esz_bp = 0;
 370	return quota;
 371}
 372
 373struct damos *damon_new_scheme(struct damos_access_pattern *pattern,
 374			enum damos_action action,
 375			unsigned long apply_interval_us,
 376			struct damos_quota *quota,
 377			struct damos_watermarks *wmarks,
 378			int target_nid)
 379{
 380	struct damos *scheme;
 381
 382	scheme = kmalloc(sizeof(*scheme), GFP_KERNEL);
 383	if (!scheme)
 384		return NULL;
 385	scheme->pattern = *pattern;
 386	scheme->action = action;
 387	scheme->apply_interval_us = apply_interval_us;
 388	/*
 389	 * next_apply_sis will be set when kdamond starts.  While kdamond is
 390	 * running, it will also updated when it is added to the DAMON context,
 391	 * or damon_attrs are updated.
 392	 */
 393	scheme->next_apply_sis = 0;
 394	scheme->walk_completed = false;
 395	INIT_LIST_HEAD(&scheme->core_filters);
 396	INIT_LIST_HEAD(&scheme->ops_filters);
 397	scheme->stat = (struct damos_stat){};
 398	INIT_LIST_HEAD(&scheme->list);
 399
 400	scheme->quota = *(damos_quota_init(quota));
 401	/* quota.goals should be separately set by caller */
 402	INIT_LIST_HEAD(&scheme->quota.goals);
 403
 404	scheme->wmarks = *wmarks;
 405	scheme->wmarks.activated = true;
 406
 407	scheme->migrate_dests = (struct damos_migrate_dests){};
 408	scheme->target_nid = target_nid;
 409
 410	return scheme;
 411}
 412
 413static void damos_set_next_apply_sis(struct damos *s, struct damon_ctx *ctx)
 414{
 415	unsigned long sample_interval = ctx->attrs.sample_interval ?
 416		ctx->attrs.sample_interval : 1;
 417	unsigned long apply_interval = s->apply_interval_us ?
 418		s->apply_interval_us : ctx->attrs.aggr_interval;
 419
 420	s->next_apply_sis = ctx->passed_sample_intervals +
 421		apply_interval / sample_interval;
 422}
 423
 424void damon_add_scheme(struct damon_ctx *ctx, struct damos *s)
 425{
 426	list_add_tail(&s->list, &ctx->schemes);
 427	damos_set_next_apply_sis(s, ctx);
 428}
 429
 430static void damon_del_scheme(struct damos *s)
 431{
 432	list_del(&s->list);
 433}
 434
 435static void damon_free_scheme(struct damos *s)
 436{
 437	kfree(s);
 438}
 439
 440void damon_destroy_scheme(struct damos *s)
 441{
 442	struct damos_quota_goal *g, *g_next;
 443	struct damos_filter *f, *next;
 444
 445	damos_for_each_quota_goal_safe(g, g_next, &s->quota)
 446		damos_destroy_quota_goal(g);
 447
 448	damos_for_each_core_filter_safe(f, next, s)
 449		damos_destroy_filter(f);
 450
 451	damos_for_each_ops_filter_safe(f, next, s)
 452		damos_destroy_filter(f);
 453
 454	kfree(s->migrate_dests.node_id_arr);
 455	kfree(s->migrate_dests.weight_arr);
 456	damon_del_scheme(s);
 457	damon_free_scheme(s);
 458}
 459
 460/*
 461 * Construct a damon_target struct
 462 *
 463 * Returns the pointer to the new struct if success, or NULL otherwise
 464 */
 465struct damon_target *damon_new_target(void)
 466{
 467	struct damon_target *t;
 468
 469	t = kmalloc(sizeof(*t), GFP_KERNEL);
 470	if (!t)
 471		return NULL;
 472
 473	t->pid = NULL;
 474	t->nr_regions = 0;
 475	INIT_LIST_HEAD(&t->regions_list);
 476	INIT_LIST_HEAD(&t->list);
 477	t->obsolete = false;
 478
 479	return t;
 480}
 481
 482void damon_add_target(struct damon_ctx *ctx, struct damon_target *t)
 483{
 484	list_add_tail(&t->list, &ctx->adaptive_targets);
 485}
 486
 487bool damon_targets_empty(struct damon_ctx *ctx)
 488{
 489	return list_empty(&ctx->adaptive_targets);
 490}
 491
 492static void damon_del_target(struct damon_target *t)
 493{
 494	list_del(&t->list);
 495}
 496
 497void damon_free_target(struct damon_target *t)
 498{
 499	struct damon_region *r, *next;
 500
 501	damon_for_each_region_safe(r, next, t)
 502		damon_free_region(r);
 503	kfree(t);
 504}
 505
 506void damon_destroy_target(struct damon_target *t, struct damon_ctx *ctx)
 507{
 508
 509	if (ctx && ctx->ops.cleanup_target)
 510		ctx->ops.cleanup_target(t);
 511
 512	damon_del_target(t);
 513	damon_free_target(t);
 514}
 515
 516unsigned int damon_nr_regions(struct damon_target *t)
 517{
 518	return t->nr_regions;
 519}
 520
 521struct damon_ctx *damon_new_ctx(void)
 522{
 523	struct damon_ctx *ctx;
 524
 525	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 526	if (!ctx)
 527		return NULL;
 528
 529	init_completion(&ctx->kdamond_started);
 530
 531	ctx->attrs.sample_interval = 5 * 1000;
 532	ctx->attrs.aggr_interval = 100 * 1000;
 533	ctx->attrs.ops_update_interval = 60 * 1000 * 1000;
 534
 535	ctx->passed_sample_intervals = 0;
 536	/* These will be set from kdamond_init_ctx() */
 537	ctx->next_aggregation_sis = 0;
 538	ctx->next_ops_update_sis = 0;
 539
 540	mutex_init(&ctx->kdamond_lock);
 541	INIT_LIST_HEAD(&ctx->call_controls);
 542	mutex_init(&ctx->call_controls_lock);
 543	mutex_init(&ctx->walk_control_lock);
 544
 545	ctx->attrs.min_nr_regions = 10;
 546	ctx->attrs.max_nr_regions = 1000;
 547
 548	ctx->addr_unit = 1;
 549	ctx->min_region_sz = DAMON_MIN_REGION_SZ;
 550
 551	INIT_LIST_HEAD(&ctx->adaptive_targets);
 552	INIT_LIST_HEAD(&ctx->schemes);
 553
 554	return ctx;
 555}
 556
 557static void damon_destroy_targets(struct damon_ctx *ctx)
 558{
 559	struct damon_target *t, *next_t;
 560
 561	damon_for_each_target_safe(t, next_t, ctx)
 562		damon_destroy_target(t, ctx);
 563}
 564
 565void damon_destroy_ctx(struct damon_ctx *ctx)
 566{
 567	struct damos *s, *next_s;
 568
 569	damon_destroy_targets(ctx);
 570
 571	damon_for_each_scheme_safe(s, next_s, ctx)
 572		damon_destroy_scheme(s);
 573
 574	kfree(ctx);
 575}
 576
 577static bool damon_attrs_equals(const struct damon_attrs *attrs1,
 578		const struct damon_attrs *attrs2)
 579{
 580	const struct damon_intervals_goal *ig1 = &attrs1->intervals_goal;
 581	const struct damon_intervals_goal *ig2 = &attrs2->intervals_goal;
 582
 583	return attrs1->sample_interval == attrs2->sample_interval &&
 584		attrs1->aggr_interval == attrs2->aggr_interval &&
 585		attrs1->ops_update_interval == attrs2->ops_update_interval &&
 586		attrs1->min_nr_regions == attrs2->min_nr_regions &&
 587		attrs1->max_nr_regions == attrs2->max_nr_regions &&
 588		ig1->access_bp == ig2->access_bp &&
 589		ig1->aggrs == ig2->aggrs &&
 590		ig1->min_sample_us == ig2->min_sample_us &&
 591		ig1->max_sample_us == ig2->max_sample_us;
 592}
 593
 594static unsigned int damon_age_for_new_attrs(unsigned int age,
 595		struct damon_attrs *old_attrs, struct damon_attrs *new_attrs)
 596{
 597	return age * old_attrs->aggr_interval / new_attrs->aggr_interval;
 598}
 599
 600/* convert access ratio in bp (per 10,000) to nr_accesses */
 601static unsigned int damon_accesses_bp_to_nr_accesses(
 602		unsigned int accesses_bp, struct damon_attrs *attrs)
 603{
 604	return accesses_bp * damon_max_nr_accesses(attrs) / 10000;
 605}
 606
 607/*
 608 * Convert nr_accesses to access ratio in bp (per 10,000).
 609 *
 610 * Callers should ensure attrs.aggr_interval is not zero, like
 611 * damon_update_monitoring_results() does .  Otherwise, divide-by-zero would
 612 * happen.
 613 */
 614static unsigned int damon_nr_accesses_to_accesses_bp(
 615		unsigned int nr_accesses, struct damon_attrs *attrs)
 616{
 617	return nr_accesses * 10000 / damon_max_nr_accesses(attrs);
 618}
 619
 620static unsigned int damon_nr_accesses_for_new_attrs(unsigned int nr_accesses,
 621		struct damon_attrs *old_attrs, struct damon_attrs *new_attrs)
 622{
 623	return damon_accesses_bp_to_nr_accesses(
 624			damon_nr_accesses_to_accesses_bp(
 625				nr_accesses, old_attrs),
 626			new_attrs);
 627}
 628
 629static void damon_update_monitoring_result(struct damon_region *r,
 630		struct damon_attrs *old_attrs, struct damon_attrs *new_attrs,
 631		bool aggregating)
 632{
 633	if (!aggregating) {
 634		r->nr_accesses = damon_nr_accesses_for_new_attrs(
 635				r->nr_accesses, old_attrs, new_attrs);
 636		r->nr_accesses_bp = r->nr_accesses * 10000;
 637	} else {
 638		/*
 639		 * if this is called in the middle of the aggregation, reset
 640		 * the aggregations we made so far for this aggregation
 641		 * interval.  In other words, make the status like
 642		 * kdamond_reset_aggregated() is called.
 643		 */
 644		r->last_nr_accesses = damon_nr_accesses_for_new_attrs(
 645				r->last_nr_accesses, old_attrs, new_attrs);
 646		r->nr_accesses_bp = r->last_nr_accesses * 10000;
 647		r->nr_accesses = 0;
 648	}
 649	r->age = damon_age_for_new_attrs(r->age, old_attrs, new_attrs);
 650}
 651
 652/*
 653 * region->nr_accesses is the number of sampling intervals in the last
 654 * aggregation interval that access to the region has found, and region->age is
 655 * the number of aggregation intervals that its access pattern has maintained.
 656 * For the reason, the real meaning of the two fields depend on current
 657 * sampling interval and aggregation interval.  This function updates
 658 * ->nr_accesses and ->age of given damon_ctx's regions for new damon_attrs.
 659 */
 660static void damon_update_monitoring_results(struct damon_ctx *ctx,
 661		struct damon_attrs *new_attrs, bool aggregating)
 662{
 663	struct damon_attrs *old_attrs = &ctx->attrs;
 664	struct damon_target *t;
 665	struct damon_region *r;
 666
 667	/* if any interval is zero, simply forgive conversion */
 668	if (!old_attrs->sample_interval || !old_attrs->aggr_interval ||
 669			!new_attrs->sample_interval ||
 670			!new_attrs->aggr_interval)
 671		return;
 672
 673	damon_for_each_target(t, ctx)
 674		damon_for_each_region(r, t)
 675			damon_update_monitoring_result(
 676					r, old_attrs, new_attrs, aggregating);
 677}
 678
 679/*
 680 * damon_valid_intervals_goal() - return if the intervals goal of @attrs is
 681 * valid.
 682 */
 683static bool damon_valid_intervals_goal(struct damon_attrs *attrs)
 684{
 685	struct damon_intervals_goal *goal = &attrs->intervals_goal;
 686
 687	/* tuning is disabled */
 688	if (!goal->aggrs)
 689		return true;
 690	if (goal->min_sample_us > goal->max_sample_us)
 691		return false;
 692	if (attrs->sample_interval < goal->min_sample_us ||
 693			goal->max_sample_us < attrs->sample_interval)
 694		return false;
 695	return true;
 696}
 697
 698/**
 699 * damon_set_attrs() - Set attributes for the monitoring.
 700 * @ctx:		monitoring context
 701 * @attrs:		monitoring attributes
 702 *
 703 * This function should be called while the kdamond is not running, an access
 704 * check results aggregation is not ongoing (e.g., from damon_call().
 705 *
 706 * Every time interval is in micro-seconds.
 707 *
 708 * Return: 0 on success, negative error code otherwise.
 709 */
 710int damon_set_attrs(struct damon_ctx *ctx, struct damon_attrs *attrs)
 711{
 712	unsigned long sample_interval = attrs->sample_interval ?
 713		attrs->sample_interval : 1;
 714	struct damos *s;
 715	bool aggregating = ctx->passed_sample_intervals <
 716		ctx->next_aggregation_sis;
 717
 718	if (!damon_valid_intervals_goal(attrs))
 719		return -EINVAL;
 720
 721	if (attrs->min_nr_regions < 3)
 722		return -EINVAL;
 723	if (attrs->min_nr_regions > attrs->max_nr_regions)
 724		return -EINVAL;
 725	if (attrs->sample_interval > attrs->aggr_interval)
 726		return -EINVAL;
 727
 728	/* calls from core-external doesn't set this. */
 729	if (!attrs->aggr_samples)
 730		attrs->aggr_samples = attrs->aggr_interval / sample_interval;
 731
 732	ctx->next_aggregation_sis = ctx->passed_sample_intervals +
 733		attrs->aggr_interval / sample_interval;
 734	ctx->next_ops_update_sis = ctx->passed_sample_intervals +
 735		attrs->ops_update_interval / sample_interval;
 736
 737	damon_update_monitoring_results(ctx, attrs, aggregating);
 738	ctx->attrs = *attrs;
 739
 740	damon_for_each_scheme(s, ctx)
 741		damos_set_next_apply_sis(s, ctx);
 742
 743	return 0;
 744}
 745
 746/**
 747 * damon_set_schemes() - Set data access monitoring based operation schemes.
 748 * @ctx:	monitoring context
 749 * @schemes:	array of the schemes
 750 * @nr_schemes:	number of entries in @schemes
 751 *
 752 * This function should not be called while the kdamond of the context is
 753 * running.
 754 */
 755void damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes,
 756			ssize_t nr_schemes)
 757{
 758	struct damos *s, *next;
 759	ssize_t i;
 760
 761	damon_for_each_scheme_safe(s, next, ctx)
 762		damon_destroy_scheme(s);
 763	for (i = 0; i < nr_schemes; i++)
 764		damon_add_scheme(ctx, schemes[i]);
 765}
 766
 767static struct damos_quota_goal *damos_nth_quota_goal(
 768		int n, struct damos_quota *q)
 769{
 770	struct damos_quota_goal *goal;
 771	int i = 0;
 772
 773	damos_for_each_quota_goal(goal, q) {
 774		if (i++ == n)
 775			return goal;
 776	}
 777	return NULL;
 778}
 779
 780static void damos_commit_quota_goal_union(
 781		struct damos_quota_goal *dst, struct damos_quota_goal *src)
 782{
 783	switch (dst->metric) {
 784	case DAMOS_QUOTA_NODE_MEM_USED_BP:
 785	case DAMOS_QUOTA_NODE_MEM_FREE_BP:
 786		dst->nid = src->nid;
 787		break;
 788	case DAMOS_QUOTA_NODE_MEMCG_USED_BP:
 789	case DAMOS_QUOTA_NODE_MEMCG_FREE_BP:
 790		dst->nid = src->nid;
 791		dst->memcg_id = src->memcg_id;
 792		break;
 793	default:
 794		break;
 795	}
 796}
 797
 798static void damos_commit_quota_goal(
 799		struct damos_quota_goal *dst, struct damos_quota_goal *src)
 800{
 801	dst->metric = src->metric;
 802	dst->target_value = src->target_value;
 803	if (dst->metric == DAMOS_QUOTA_USER_INPUT)
 804		dst->current_value = src->current_value;
 805	/* keep last_psi_total as is, since it will be updated in next cycle */
 806	damos_commit_quota_goal_union(dst, src);
 807}
 808
 809/**
 810 * damos_commit_quota_goals() - Commit DAMOS quota goals to another quota.
 811 * @dst:	The commit destination DAMOS quota.
 812 * @src:	The commit source DAMOS quota.
 813 *
 814 * Copies user-specified parameters for quota goals from @src to @dst.  Users
 815 * should use this function for quota goals-level parameters update of running
 816 * DAMON contexts, instead of manual in-place updates.
 817 *
 818 * This function should be called from parameters-update safe context, like
 819 * damon_call().
 820 */
 821int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src)
 822{
 823	struct damos_quota_goal *dst_goal, *next, *src_goal, *new_goal;
 824	int i = 0, j = 0;
 825
 826	damos_for_each_quota_goal_safe(dst_goal, next, dst) {
 827		src_goal = damos_nth_quota_goal(i++, src);
 828		if (src_goal)
 829			damos_commit_quota_goal(dst_goal, src_goal);
 830		else
 831			damos_destroy_quota_goal(dst_goal);
 832	}
 833	damos_for_each_quota_goal_safe(src_goal, next, src) {
 834		if (j++ < i)
 835			continue;
 836		new_goal = damos_new_quota_goal(
 837				src_goal->metric, src_goal->target_value);
 838		if (!new_goal)
 839			return -ENOMEM;
 840		damos_commit_quota_goal(new_goal, src_goal);
 841		damos_add_quota_goal(dst, new_goal);
 842	}
 843	return 0;
 844}
 845
 846static int damos_commit_quota(struct damos_quota *dst, struct damos_quota *src)
 847{
 848	int err;
 849
 850	dst->reset_interval = src->reset_interval;
 851	dst->ms = src->ms;
 852	dst->sz = src->sz;
 853	err = damos_commit_quota_goals(dst, src);
 854	if (err)
 855		return err;
 856	dst->weight_sz = src->weight_sz;
 857	dst->weight_nr_accesses = src->weight_nr_accesses;
 858	dst->weight_age = src->weight_age;
 859	return 0;
 860}
 861
 862static struct damos_filter *damos_nth_core_filter(int n, struct damos *s)
 863{
 864	struct damos_filter *filter;
 865	int i = 0;
 866
 867	damos_for_each_core_filter(filter, s) {
 868		if (i++ == n)
 869			return filter;
 870	}
 871	return NULL;
 872}
 873
 874static struct damos_filter *damos_nth_ops_filter(int n, struct damos *s)
 875{
 876	struct damos_filter *filter;
 877	int i = 0;
 878
 879	damos_for_each_ops_filter(filter, s) {
 880		if (i++ == n)
 881			return filter;
 882	}
 883	return NULL;
 884}
 885
 886static void damos_commit_filter_arg(
 887		struct damos_filter *dst, struct damos_filter *src)
 888{
 889	switch (dst->type) {
 890	case DAMOS_FILTER_TYPE_MEMCG:
 891		dst->memcg_id = src->memcg_id;
 892		break;
 893	case DAMOS_FILTER_TYPE_ADDR:
 894		dst->addr_range = src->addr_range;
 895		break;
 896	case DAMOS_FILTER_TYPE_TARGET:
 897		dst->target_idx = src->target_idx;
 898		break;
 899	case DAMOS_FILTER_TYPE_HUGEPAGE_SIZE:
 900		dst->sz_range = src->sz_range;
 901		break;
 902	default:
 903		break;
 904	}
 905}
 906
 907static void damos_commit_filter(
 908		struct damos_filter *dst, struct damos_filter *src)
 909{
 910	dst->type = src->type;
 911	dst->matching = src->matching;
 912	dst->allow = src->allow;
 913	damos_commit_filter_arg(dst, src);
 914}
 915
 916static int damos_commit_core_filters(struct damos *dst, struct damos *src)
 917{
 918	struct damos_filter *dst_filter, *next, *src_filter, *new_filter;
 919	int i = 0, j = 0;
 920
 921	damos_for_each_core_filter_safe(dst_filter, next, dst) {
 922		src_filter = damos_nth_core_filter(i++, src);
 923		if (src_filter)
 924			damos_commit_filter(dst_filter, src_filter);
 925		else
 926			damos_destroy_filter(dst_filter);
 927	}
 928
 929	damos_for_each_core_filter_safe(src_filter, next, src) {
 930		if (j++ < i)
 931			continue;
 932
 933		new_filter = damos_new_filter(
 934				src_filter->type, src_filter->matching,
 935				src_filter->allow);
 936		if (!new_filter)
 937			return -ENOMEM;
 938		damos_commit_filter_arg(new_filter, src_filter);
 939		damos_add_filter(dst, new_filter);
 940	}
 941	return 0;
 942}
 943
 944static int damos_commit_ops_filters(struct damos *dst, struct damos *src)
 945{
 946	struct damos_filter *dst_filter, *next, *src_filter, *new_filter;
 947	int i = 0, j = 0;
 948
 949	damos_for_each_ops_filter_safe(dst_filter, next, dst) {
 950		src_filter = damos_nth_ops_filter(i++, src);
 951		if (src_filter)
 952			damos_commit_filter(dst_filter, src_filter);
 953		else
 954			damos_destroy_filter(dst_filter);
 955	}
 956
 957	damos_for_each_ops_filter_safe(src_filter, next, src) {
 958		if (j++ < i)
 959			continue;
 960
 961		new_filter = damos_new_filter(
 962				src_filter->type, src_filter->matching,
 963				src_filter->allow);
 964		if (!new_filter)
 965			return -ENOMEM;
 966		damos_commit_filter_arg(new_filter, src_filter);
 967		damos_add_filter(dst, new_filter);
 968	}
 969	return 0;
 970}
 971
 972/**
 973 * damos_filters_default_reject() - decide whether to reject memory that didn't
 974 *				    match with any given filter.
 975 * @filters:	Given DAMOS filters of a group.
 976 */
 977static bool damos_filters_default_reject(struct list_head *filters)
 978{
 979	struct damos_filter *last_filter;
 980
 981	if (list_empty(filters))
 982		return false;
 983	last_filter = list_last_entry(filters, struct damos_filter, list);
 984	return last_filter->allow;
 985}
 986
 987static void damos_set_filters_default_reject(struct damos *s)
 988{
 989	if (!list_empty(&s->ops_filters))
 990		s->core_filters_default_reject = false;
 991	else
 992		s->core_filters_default_reject =
 993			damos_filters_default_reject(&s->core_filters);
 994	s->ops_filters_default_reject =
 995		damos_filters_default_reject(&s->ops_filters);
 996}
 997
 998static int damos_commit_dests(struct damos_migrate_dests *dst,
 999		struct damos_migrate_dests *src)
1000{
1001	if (dst->nr_dests != src->nr_dests) {
1002		kfree(dst->node_id_arr);
1003		kfree(dst->weight_arr);
1004
1005		dst->node_id_arr = kmalloc_array(src->nr_dests,
1006			sizeof(*dst->node_id_arr), GFP_KERNEL);
1007		if (!dst->node_id_arr) {
1008			dst->weight_arr = NULL;
1009			return -ENOMEM;
1010		}
1011
1012		dst->weight_arr = kmalloc_array(src->nr_dests,
1013			sizeof(*dst->weight_arr), GFP_KERNEL);
1014		if (!dst->weight_arr) {
1015			/* ->node_id_arr will be freed by scheme destruction */
1016			return -ENOMEM;
1017		}
1018	}
1019
1020	dst->nr_dests = src->nr_dests;
1021	for (int i = 0; i < src->nr_dests; i++) {
1022		dst->node_id_arr[i] = src->node_id_arr[i];
1023		dst->weight_arr[i] = src->weight_arr[i];
1024	}
1025
1026	return 0;
1027}
1028
1029static int damos_commit_filters(struct damos *dst, struct damos *src)
1030{
1031	int err;
1032
1033	err = damos_commit_core_filters(dst, src);
1034	if (err)
1035		return err;
1036	err = damos_commit_ops_filters(dst, src);
1037	if (err)
1038		return err;
1039	damos_set_filters_default_reject(dst);
1040	return 0;
1041}
1042
1043static struct damos *damon_nth_scheme(int n, struct damon_ctx *ctx)
1044{
1045	struct damos *s;
1046	int i = 0;
1047
1048	damon_for_each_scheme(s, ctx) {
1049		if (i++ == n)
1050			return s;
1051	}
1052	return NULL;
1053}
1054
1055static int damos_commit(struct damos *dst, struct damos *src)
1056{
1057	int err;
1058
1059	dst->pattern = src->pattern;
1060	dst->action = src->action;
1061	dst->apply_interval_us = src->apply_interval_us;
1062
1063	err = damos_commit_quota(&dst->quota, &src->quota);
1064	if (err)
1065		return err;
1066
1067	dst->wmarks = src->wmarks;
1068	dst->target_nid = src->target_nid;
1069
1070	err = damos_commit_dests(&dst->migrate_dests, &src->migrate_dests);
1071	if (err)
1072		return err;
1073
1074	err = damos_commit_filters(dst, src);
1075	return err;
1076}
1077
1078static int damon_commit_schemes(struct damon_ctx *dst, struct damon_ctx *src)
1079{
1080	struct damos *dst_scheme, *next, *src_scheme, *new_scheme;
1081	int i = 0, j = 0, err;
1082
1083	damon_for_each_scheme_safe(dst_scheme, next, dst) {
1084		src_scheme = damon_nth_scheme(i++, src);
1085		if (src_scheme) {
1086			err = damos_commit(dst_scheme, src_scheme);
1087			if (err)
1088				return err;
1089		} else {
1090			damon_destroy_scheme(dst_scheme);
1091		}
1092	}
1093
1094	damon_for_each_scheme_safe(src_scheme, next, src) {
1095		if (j++ < i)
1096			continue;
1097		new_scheme = damon_new_scheme(&src_scheme->pattern,
1098				src_scheme->action,
1099				src_scheme->apply_interval_us,
1100				&src_scheme->quota, &src_scheme->wmarks,
1101				NUMA_NO_NODE);
1102		if (!new_scheme)
1103			return -ENOMEM;
1104		err = damos_commit(new_scheme, src_scheme);
1105		if (err) {
1106			damon_destroy_scheme(new_scheme);
1107			return err;
1108		}
1109		damon_add_scheme(dst, new_scheme);
1110	}
1111	return 0;
1112}
1113
1114static struct damon_target *damon_nth_target(int n, struct damon_ctx *ctx)
1115{
1116	struct damon_target *t;
1117	int i = 0;
1118
1119	damon_for_each_target(t, ctx) {
1120		if (i++ == n)
1121			return t;
1122	}
1123	return NULL;
1124}
1125
1126/*
1127 * The caller should ensure the regions of @src are
1128 * 1. valid (end >= src) and
1129 * 2. sorted by starting address.
1130 *
1131 * If @src has no region, @dst keeps current regions.
1132 */
1133static int damon_commit_target_regions(struct damon_target *dst,
1134		struct damon_target *src, unsigned long src_min_region_sz)
1135{
1136	struct damon_region *src_region;
1137	struct damon_addr_range *ranges;
1138	int i = 0, err;
1139
1140	damon_for_each_region(src_region, src)
1141		i++;
1142	if (!i)
1143		return 0;
1144
1145	ranges = kmalloc_array(i, sizeof(*ranges), GFP_KERNEL | __GFP_NOWARN);
1146	if (!ranges)
1147		return -ENOMEM;
1148	i = 0;
1149	damon_for_each_region(src_region, src)
1150		ranges[i++] = src_region->ar;
1151	err = damon_set_regions(dst, ranges, i, src_min_region_sz);
1152	kfree(ranges);
1153	return err;
1154}
1155
1156static int damon_commit_target(
1157		struct damon_target *dst, bool dst_has_pid,
1158		struct damon_target *src, bool src_has_pid,
1159		unsigned long src_min_region_sz)
1160{
1161	int err;
1162
1163	err = damon_commit_target_regions(dst, src, src_min_region_sz);
1164	if (err)
1165		return err;
1166	if (dst_has_pid)
1167		put_pid(dst->pid);
1168	if (src_has_pid)
1169		get_pid(src->pid);
1170	dst->pid = src->pid;
1171	return 0;
1172}
1173
1174static int damon_commit_targets(
1175		struct damon_ctx *dst, struct damon_ctx *src)
1176{
1177	struct damon_target *dst_target, *next, *src_target, *new_target;
1178	int i = 0, j = 0, err;
1179
1180	damon_for_each_target_safe(dst_target, next, dst) {
1181		src_target = damon_nth_target(i++, src);
1182		/*
1183		 * If src target is obsolete, do not commit the parameters to
1184		 * the dst target, and further remove the dst target.
1185		 */
1186		if (src_target && !src_target->obsolete) {
1187			err = damon_commit_target(
1188					dst_target, damon_target_has_pid(dst),
1189					src_target, damon_target_has_pid(src),
1190					src->min_region_sz);
1191			if (err)
1192				return err;
1193		} else {
1194			struct damos *s;
1195
1196			damon_destroy_target(dst_target, dst);
1197			damon_for_each_scheme(s, dst) {
1198				if (s->quota.charge_target_from == dst_target) {
1199					s->quota.charge_target_from = NULL;
1200					s->quota.charge_addr_from = 0;
1201				}
1202			}
1203		}
1204	}
1205
1206	damon_for_each_target_safe(src_target, next, src) {
1207		if (j++ < i)
1208			continue;
1209		/* target to remove has no matching dst */
1210		if (src_target->obsolete)
1211			return -EINVAL;
1212		new_target = damon_new_target();
1213		if (!new_target)
1214			return -ENOMEM;
1215		err = damon_commit_target(new_target, false,
1216				src_target, damon_target_has_pid(src),
1217				src->min_region_sz);
1218		if (err) {
1219			damon_destroy_target(new_target, NULL);
1220			return err;
1221		}
1222		damon_add_target(dst, new_target);
1223	}
1224	return 0;
1225}
1226
1227/**
1228 * damon_commit_ctx() - Commit parameters of a DAMON context to another.
1229 * @dst:	The commit destination DAMON context.
1230 * @src:	The commit source DAMON context.
1231 *
1232 * This function copies user-specified parameters from @src to @dst and update
1233 * the internal status and results accordingly.  Users should use this function
1234 * for context-level parameters update of running context, instead of manual
1235 * in-place updates.
1236 *
1237 * This function should be called from parameters-update safe context, like
1238 * damon_call().
1239 */
1240int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src)
1241{
1242	int err;
1243
1244	dst->maybe_corrupted = true;
1245	if (!is_power_of_2(src->min_region_sz))
1246		return -EINVAL;
1247
1248	err = damon_commit_schemes(dst, src);
1249	if (err)
1250		return err;
1251	err = damon_commit_targets(dst, src);
1252	if (err)
1253		return err;
1254	/*
1255	 * schemes and targets should be updated first, since
1256	 * 1. damon_set_attrs() updates monitoring results of targets and
1257	 * next_apply_sis of schemes, and
1258	 * 2. ops update should be done after pid handling is done (target
1259	 *    committing require putting pids).
1260	 */
1261	if (!damon_attrs_equals(&dst->attrs, &src->attrs)) {
1262		err = damon_set_attrs(dst, &src->attrs);
1263		if (err)
1264			return err;
1265	}
1266	dst->ops = src->ops;
1267	dst->addr_unit = src->addr_unit;
1268	dst->min_region_sz = src->min_region_sz;
1269
1270	dst->maybe_corrupted = false;
1271	return 0;
1272}
1273
1274/**
1275 * damon_nr_running_ctxs() - Return number of currently running contexts.
1276 */
1277int damon_nr_running_ctxs(void)
1278{
1279	int nr_ctxs;
1280
1281	mutex_lock(&damon_lock);
1282	nr_ctxs = nr_running_ctxs;
1283	mutex_unlock(&damon_lock);
1284
1285	return nr_ctxs;
1286}
1287
1288/* Returns the size upper limit for each monitoring region */
1289static unsigned long damon_region_sz_limit(struct damon_ctx *ctx)
1290{
1291	struct damon_target *t;
1292	struct damon_region *r;
1293	unsigned long sz = 0;
1294
1295	damon_for_each_target(t, ctx) {
1296		damon_for_each_region(r, t)
1297			sz += damon_sz_region(r);
1298	}
1299
1300	if (ctx->attrs.min_nr_regions)
1301		sz /= ctx->attrs.min_nr_regions;
1302	if (sz < ctx->min_region_sz)
1303		sz = ctx->min_region_sz;
1304
1305	return sz;
1306}
1307
1308static int kdamond_fn(void *data);
1309
1310/*
1311 * __damon_start() - Starts monitoring with given context.
1312 * @ctx:	monitoring context
1313 *
1314 * This function should be called while damon_lock is hold.
1315 *
1316 * Return: 0 on success, negative error code otherwise.
1317 */
1318static int __damon_start(struct damon_ctx *ctx)
1319{
1320	int err = -EBUSY;
1321
1322	mutex_lock(&ctx->kdamond_lock);
1323	if (!ctx->kdamond) {
1324		err = 0;
1325		reinit_completion(&ctx->kdamond_started);
1326		ctx->kdamond = kthread_run(kdamond_fn, ctx, "kdamond.%d",
1327				nr_running_ctxs);
1328		if (IS_ERR(ctx->kdamond)) {
1329			err = PTR_ERR(ctx->kdamond);
1330			ctx->kdamond = NULL;
1331		} else {
1332			wait_for_completion(&ctx->kdamond_started);
1333		}
1334	}
1335	mutex_unlock(&ctx->kdamond_lock);
1336
1337	return err;
1338}
1339
1340/**
1341 * damon_start() - Starts the monitorings for a given group of contexts.
1342 * @ctxs:	an array of the pointers for contexts to start monitoring
1343 * @nr_ctxs:	size of @ctxs
1344 * @exclusive:	exclusiveness of this contexts group
1345 *
1346 * This function starts a group of monitoring threads for a group of monitoring
1347 * contexts.  One thread per each context is created and run in parallel.  The
1348 * caller should handle synchronization between the threads by itself.  If
1349 * @exclusive is true and a group of threads that created by other
1350 * 'damon_start()' call is currently running, this function does nothing but
1351 * returns -EBUSY.
1352 *
1353 * Return: 0 on success, negative error code otherwise.
1354 */
1355int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive)
1356{
1357	int i;
1358	int err = 0;
1359
1360	mutex_lock(&damon_lock);
1361	if ((exclusive && nr_running_ctxs) ||
1362			(!exclusive && running_exclusive_ctxs)) {
1363		mutex_unlock(&damon_lock);
1364		return -EBUSY;
1365	}
1366
1367	for (i = 0; i < nr_ctxs; i++) {
1368		err = __damon_start(ctxs[i]);
1369		if (err)
1370			break;
1371		nr_running_ctxs++;
1372	}
1373	if (exclusive && nr_running_ctxs)
1374		running_exclusive_ctxs = true;
1375	mutex_unlock(&damon_lock);
1376
1377	return err;
1378}
1379
1380/*
1381 * __damon_stop() - Stops monitoring of a given context.
1382 * @ctx:	monitoring context
1383 *
1384 * Return: 0 on success, negative error code otherwise.
1385 */
1386static int __damon_stop(struct damon_ctx *ctx)
1387{
1388	struct task_struct *tsk;
1389
1390	mutex_lock(&ctx->kdamond_lock);
1391	tsk = ctx->kdamond;
1392	if (tsk) {
1393		get_task_struct(tsk);
1394		mutex_unlock(&ctx->kdamond_lock);
1395		kthread_stop_put(tsk);
1396		return 0;
1397	}
1398	mutex_unlock(&ctx->kdamond_lock);
1399
1400	return -EPERM;
1401}
1402
1403/**
1404 * damon_stop() - Stops the monitorings for a given group of contexts.
1405 * @ctxs:	an array of the pointers for contexts to stop monitoring
1406 * @nr_ctxs:	size of @ctxs
1407 *
1408 * Return: 0 on success, negative error code otherwise.
1409 */
1410int damon_stop(struct damon_ctx **ctxs, int nr_ctxs)
1411{
1412	int i, err = 0;
1413
1414	for (i = 0; i < nr_ctxs; i++) {
1415		/* nr_running_ctxs is decremented in kdamond_fn */
1416		err = __damon_stop(ctxs[i]);
1417		if (err)
1418			break;
1419	}
1420	return err;
1421}
1422
1423/**
1424 * damon_is_running() - Returns if a given DAMON context is running.
1425 * @ctx:	The DAMON context to see if running.
1426 *
1427 * Return: true if @ctx is running, false otherwise.
1428 */
1429bool damon_is_running(struct damon_ctx *ctx)
1430{
1431	bool running;
1432
1433	mutex_lock(&ctx->kdamond_lock);
1434	running = ctx->kdamond != NULL;
1435	mutex_unlock(&ctx->kdamond_lock);
1436	return running;
1437}
1438
1439/*
1440 * damon_call_handle_inactive_ctx() - handle DAMON call request that added to
1441 *				      an inactive context.
1442 * @ctx:	The inactive DAMON context.
1443 * @control:	Control variable of the call request.
1444 *
1445 * This function is called in a case that @control is added to @ctx but @ctx is
1446 * not running (inactive).  See if @ctx handled @control or not, and cleanup
1447 * @control if it was not handled.
1448 *
1449 * Returns 0 if @control was handled by @ctx, negative error code otherwise.
1450 */
1451static int damon_call_handle_inactive_ctx(
1452		struct damon_ctx *ctx, struct damon_call_control *control)
1453{
1454	struct damon_call_control *c;
1455
1456	mutex_lock(&ctx->call_controls_lock);
1457	list_for_each_entry(c, &ctx->call_controls, list) {
1458		if (c == control) {
1459			list_del(&control->list);
1460			mutex_unlock(&ctx->call_controls_lock);
1461			return -EINVAL;
1462		}
1463	}
1464	mutex_unlock(&ctx->call_controls_lock);
1465	return 0;
1466}
1467
1468/**
1469 * damon_call() - Invoke a given function on DAMON worker thread (kdamond).
1470 * @ctx:	DAMON context to call the function for.
1471 * @control:	Control variable of the call request.
1472 *
1473 * Ask DAMON worker thread (kdamond) of @ctx to call a function with an
1474 * argument data that respectively passed via &damon_call_control->fn and
1475 * &damon_call_control->data of @control.  If &damon_call_control->repeat of
1476 * @control is unset, further wait until the kdamond finishes handling of the
1477 * request.  Otherwise, return as soon as the request is made.
1478 *
1479 * The kdamond executes the function with the argument in the main loop, just
1480 * after a sampling of the iteration is finished.  The function can hence
1481 * safely access the internal data of the &struct damon_ctx without additional
1482 * synchronization.  The return value of the function will be saved in
1483 * &damon_call_control->return_code.
1484 *
1485 * Return: 0 on success, negative error code otherwise.
1486 */
1487int damon_call(struct damon_ctx *ctx, struct damon_call_control *control)
1488{
1489	if (!control->repeat)
1490		init_completion(&control->completion);
1491	control->canceled = false;
1492	INIT_LIST_HEAD(&control->list);
1493
1494	mutex_lock(&ctx->call_controls_lock);
1495	list_add_tail(&control->list, &ctx->call_controls);
1496	mutex_unlock(&ctx->call_controls_lock);
1497	if (!damon_is_running(ctx))
1498		return damon_call_handle_inactive_ctx(ctx, control);
1499	if (control->repeat)
1500		return 0;
1501	wait_for_completion(&control->completion);
1502	if (control->canceled)
1503		return -ECANCELED;
1504	return 0;
1505}
1506
1507/**
1508 * damos_walk() - Invoke a given functions while DAMOS walk regions.
1509 * @ctx:	DAMON context to call the functions for.
1510 * @control:	Control variable of the walk request.
1511 *
1512 * Ask DAMON worker thread (kdamond) of @ctx to call a function for each region
1513 * that the kdamond will apply DAMOS action to, and wait until the kdamond
1514 * finishes handling of the request.
1515 *
1516 * The kdamond executes the given function in the main loop, for each region
1517 * just after it applied any DAMOS actions of @ctx to it.  The invocation is
1518 * made only within one &damos->apply_interval_us since damos_walk()
1519 * invocation, for each scheme.  The given callback function can hence safely
1520 * access the internal data of &struct damon_ctx and &struct damon_region that
1521 * each of the scheme will apply the action for next interval, without
1522 * additional synchronizations against the kdamond.  If every scheme of @ctx
1523 * passed at least one &damos->apply_interval_us, kdamond marks the request as
1524 * completed so that damos_walk() can wakeup and return.
1525 *
1526 * Return: 0 on success, negative error code otherwise.
1527 */
1528int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control)
1529{
1530	init_completion(&control->completion);
1531	control->canceled = false;
1532	mutex_lock(&ctx->walk_control_lock);
1533	if (ctx->walk_control) {
1534		mutex_unlock(&ctx->walk_control_lock);
1535		return -EBUSY;
1536	}
1537	ctx->walk_control = control;
1538	mutex_unlock(&ctx->walk_control_lock);
1539	if (!damon_is_running(ctx)) {
1540		mutex_lock(&ctx->walk_control_lock);
1541		if (ctx->walk_control == control)
1542			ctx->walk_control = NULL;
1543		mutex_unlock(&ctx->walk_control_lock);
1544		return -EINVAL;
1545	}
1546	wait_for_completion(&control->completion);
1547	if (control->canceled)
1548		return -ECANCELED;
1549	return 0;
1550}
1551
1552/*
1553 * Warn and fix corrupted ->nr_accesses[_bp] for investigations and preventing
1554 * the problem being propagated.
1555 */
1556static void damon_warn_fix_nr_accesses_corruption(struct damon_region *r)
1557{
1558	if (r->nr_accesses_bp == r->nr_accesses * 10000)
1559		return;
1560	WARN_ONCE(true, "invalid nr_accesses_bp at reset: %u %u\n",
1561			r->nr_accesses_bp, r->nr_accesses);
1562	r->nr_accesses_bp = r->nr_accesses * 10000;
1563}
1564
1565/*
1566 * Reset the aggregated monitoring results ('nr_accesses' of each region).
1567 */
1568static void kdamond_reset_aggregated(struct damon_ctx *c)
1569{
1570	struct damon_target *t;
1571	unsigned int ti = 0;	/* target's index */
1572
1573	damon_for_each_target(t, c) {
1574		struct damon_region *r;
1575
1576		damon_for_each_region(r, t) {
1577			trace_damon_aggregated(ti, r, damon_nr_regions(t));
1578			damon_warn_fix_nr_accesses_corruption(r);
1579			r->last_nr_accesses = r->nr_accesses;
1580			r->nr_accesses = 0;
1581		}
1582		ti++;
1583	}
1584}
1585
1586static unsigned long damon_get_intervals_score(struct damon_ctx *c)
1587{
1588	struct damon_target *t;
1589	struct damon_region *r;
1590	unsigned long sz_region, max_access_events = 0, access_events = 0;
1591	unsigned long target_access_events;
1592	unsigned long goal_bp = c->attrs.intervals_goal.access_bp;
1593
1594	damon_for_each_target(t, c) {
1595		damon_for_each_region(r, t) {
1596			sz_region = damon_sz_region(r);
1597			max_access_events += sz_region * c->attrs.aggr_samples;
1598			access_events += sz_region * r->nr_accesses;
1599		}
1600	}
1601	target_access_events = max_access_events * goal_bp / 10000;
1602	target_access_events = target_access_events ? : 1;
1603	return access_events * 10000 / target_access_events;
1604}
1605
1606static unsigned long damon_feed_loop_next_input(unsigned long last_input,
1607		unsigned long score);
1608
1609static unsigned long damon_get_intervals_adaptation_bp(struct damon_ctx *c)
1610{
1611	unsigned long score_bp, adaptation_bp;
1612
1613	score_bp = damon_get_intervals_score(c);
1614	adaptation_bp = damon_feed_loop_next_input(100000000, score_bp) /
1615		10000;
1616	/*
1617	 * adaptaion_bp ranges from 1 to 20,000.  Avoid too rapid reduction of
1618	 * the intervals by rescaling [1,10,000] to [5000, 10,000].
1619	 */
1620	if (adaptation_bp <= 10000)
1621		adaptation_bp = 5000 + adaptation_bp / 2;
1622	return adaptation_bp;
1623}
1624
1625static void kdamond_tune_intervals(struct damon_ctx *c)
1626{
1627	unsigned long adaptation_bp;
1628	struct damon_attrs new_attrs;
1629	struct damon_intervals_goal *goal;
1630
1631	adaptation_bp = damon_get_intervals_adaptation_bp(c);
1632	if (adaptation_bp == 10000)
1633		return;
1634
1635	new_attrs = c->attrs;
1636	goal = &c->attrs.intervals_goal;
1637	new_attrs.sample_interval = min(goal->max_sample_us,
1638			c->attrs.sample_interval * adaptation_bp / 10000);
1639	new_attrs.sample_interval = max(goal->min_sample_us,
1640			new_attrs.sample_interval);
1641	new_attrs.aggr_interval = new_attrs.sample_interval *
1642		c->attrs.aggr_samples;
1643	trace_damon_monitor_intervals_tune(new_attrs.sample_interval);
1644	damon_set_attrs(c, &new_attrs);
1645}
1646
1647static void damon_split_region_at(struct damon_target *t,
1648				  struct damon_region *r, unsigned long sz_r);
1649
1650static bool __damos_valid_target(struct damon_region *r, struct damos *s)
1651{
1652	unsigned long sz;
1653	unsigned int nr_accesses = r->nr_accesses_bp / 10000;
1654
1655	sz = damon_sz_region(r);
1656	return s->pattern.min_sz_region <= sz &&
1657		sz <= s->pattern.max_sz_region &&
1658		s->pattern.min_nr_accesses <= nr_accesses &&
1659		nr_accesses <= s->pattern.max_nr_accesses &&
1660		s->pattern.min_age_region <= r->age &&
1661		r->age <= s->pattern.max_age_region;
1662}
1663
1664static bool damos_valid_target(struct damon_ctx *c, struct damon_target *t,
1665		struct damon_region *r, struct damos *s)
1666{
1667	bool ret = __damos_valid_target(r, s);
1668
1669	if (!ret || !s->quota.esz || !c->ops.get_scheme_score)
1670		return ret;
1671
1672	return c->ops.get_scheme_score(c, t, r, s) >= s->quota.min_score;
1673}
1674
1675/*
1676 * damos_skip_charged_region() - Check if the given region or starting part of
1677 * it is already charged for the DAMOS quota.
1678 * @t:	The target of the region.
1679 * @rp:	The pointer to the region.
1680 * @s:	The scheme to be applied.
1681 * @min_region_sz:	minimum region size.
1682 *
1683 * If a quota of a scheme has exceeded in a quota charge window, the scheme's
1684 * action would applied to only a part of the target access pattern fulfilling
1685 * regions.  To avoid applying the scheme action to only already applied
1686 * regions, DAMON skips applying the scheme action to the regions that charged
1687 * in the previous charge window.
1688 *
1689 * This function checks if a given region should be skipped or not for the
1690 * reason.  If only the starting part of the region has previously charged,
1691 * this function splits the region into two so that the second one covers the
1692 * area that not charged in the previous charge widnow and saves the second
1693 * region in *rp and returns false, so that the caller can apply DAMON action
1694 * to the second one.
1695 *
1696 * Return: true if the region should be entirely skipped, false otherwise.
1697 */
1698static bool damos_skip_charged_region(struct damon_target *t,
1699		struct damon_region **rp, struct damos *s,
1700		unsigned long min_region_sz)
1701{
1702	struct damon_region *r = *rp;
1703	struct damos_quota *quota = &s->quota;
1704	unsigned long sz_to_skip;
1705
1706	/* Skip previously charged regions */
1707	if (quota->charge_target_from) {
1708		if (t != quota->charge_target_from)
1709			return true;
1710		if (r == damon_last_region(t)) {
1711			quota->charge_target_from = NULL;
1712			quota->charge_addr_from = 0;
1713			return true;
1714		}
1715		if (quota->charge_addr_from &&
1716				r->ar.end <= quota->charge_addr_from)
1717			return true;
1718
1719		if (quota->charge_addr_from && r->ar.start <
1720				quota->charge_addr_from) {
1721			sz_to_skip = ALIGN_DOWN(quota->charge_addr_from -
1722					r->ar.start, min_region_sz);
1723			if (!sz_to_skip) {
1724				if (damon_sz_region(r) <= min_region_sz)
1725					return true;
1726				sz_to_skip = min_region_sz;
1727			}
1728			damon_split_region_at(t, r, sz_to_skip);
1729			r = damon_next_region(r);
1730			*rp = r;
1731		}
1732		quota->charge_target_from = NULL;
1733		quota->charge_addr_from = 0;
1734	}
1735	return false;
1736}
1737
1738static void damos_update_stat(struct damos *s,
1739		unsigned long sz_tried, unsigned long sz_applied,
1740		unsigned long sz_ops_filter_passed)
1741{
1742	s->stat.nr_tried++;
1743	s->stat.sz_tried += sz_tried;
1744	if (sz_applied)
1745		s->stat.nr_applied++;
1746	s->stat.sz_applied += sz_applied;
1747	s->stat.sz_ops_filter_passed += sz_ops_filter_passed;
1748}
1749
1750static bool damos_filter_match(struct damon_ctx *ctx, struct damon_target *t,
1751		struct damon_region *r, struct damos_filter *filter,
1752		unsigned long min_region_sz)
1753{
1754	bool matched = false;
1755	struct damon_target *ti;
1756	int target_idx = 0;
1757	unsigned long start, end;
1758
1759	switch (filter->type) {
1760	case DAMOS_FILTER_TYPE_TARGET:
1761		damon_for_each_target(ti, ctx) {
1762			if (ti == t)
1763				break;
1764			target_idx++;
1765		}
1766		matched = target_idx == filter->target_idx;
1767		break;
1768	case DAMOS_FILTER_TYPE_ADDR:
1769		start = ALIGN_DOWN(filter->addr_range.start, min_region_sz);
1770		end = ALIGN_DOWN(filter->addr_range.end, min_region_sz);
1771
1772		/* inside the range */
1773		if (start <= r->ar.start && r->ar.end <= end) {
1774			matched = true;
1775			break;
1776		}
1777		/* outside of the range */
1778		if (r->ar.end <= start || end <= r->ar.start) {
1779			matched = false;
1780			break;
1781		}
1782		/* start before the range and overlap */
1783		if (r->ar.start < start) {
1784			damon_split_region_at(t, r, start - r->ar.start);
1785			matched = false;
1786			break;
1787		}
1788		/* start inside the range */
1789		damon_split_region_at(t, r, end - r->ar.start);
1790		matched = true;
1791		break;
1792	default:
1793		return false;
1794	}
1795
1796	return matched == filter->matching;
1797}
1798
1799static bool damos_filter_out(struct damon_ctx *ctx, struct damon_target *t,
1800		struct damon_region *r, struct damos *s)
1801{
1802	struct damos_filter *filter;
1803
1804	s->core_filters_allowed = false;
1805	damos_for_each_core_filter(filter, s) {
1806		if (damos_filter_match(ctx, t, r, filter, ctx->min_region_sz)) {
1807			if (filter->allow)
1808				s->core_filters_allowed = true;
1809			return !filter->allow;
1810		}
1811	}
1812	return s->core_filters_default_reject;
1813}
1814
1815/*
1816 * damos_walk_call_walk() - Call &damos_walk_control->walk_fn.
1817 * @ctx:	The context of &damon_ctx->walk_control.
1818 * @t:		The monitoring target of @r that @s will be applied.
1819 * @r:		The region of @t that @s will be applied.
1820 * @s:		The scheme of @ctx that will be applied to @r.
1821 *
1822 * This function is called from kdamond whenever it asked the operation set to
1823 * apply a DAMOS scheme action to a region.  If a DAMOS walk request is
1824 * installed by damos_walk() and not yet uninstalled, invoke it.
1825 */
1826static void damos_walk_call_walk(struct damon_ctx *ctx, struct damon_target *t,
1827		struct damon_region *r, struct damos *s,
1828		unsigned long sz_filter_passed)
1829{
1830	struct damos_walk_control *control;
1831
1832	if (s->walk_completed)
1833		return;
1834
1835	control = ctx->walk_control;
1836	if (!control)
1837		return;
1838
1839	control->walk_fn(control->data, ctx, t, r, s, sz_filter_passed);
1840}
1841
1842/*
1843 * damos_walk_complete() - Complete DAMOS walk request if all walks are done.
1844 * @ctx:	The context of &damon_ctx->walk_control.
1845 * @s:		A scheme of @ctx that all walks are now done.
1846 *
1847 * This function is called when kdamond finished applying the action of a DAMOS
1848 * scheme to all regions that eligible for the given &damos->apply_interval_us.
1849 * If every scheme of @ctx including @s now finished walking for at least one
1850 * &damos->apply_interval_us, this function makrs the handling of the given
1851 * DAMOS walk request is done, so that damos_walk() can wake up and return.
1852 */
1853static void damos_walk_complete(struct damon_ctx *ctx, struct damos *s)
1854{
1855	struct damos *siter;
1856	struct damos_walk_control *control;
1857
1858	control = ctx->walk_control;
1859	if (!control)
1860		return;
1861
1862	s->walk_completed = true;
1863	/* if all schemes completed, signal completion to walker */
1864	damon_for_each_scheme(siter, ctx) {
1865		if (!siter->walk_completed)
1866			return;
1867	}
1868	damon_for_each_scheme(siter, ctx)
1869		siter->walk_completed = false;
1870
1871	complete(&control->completion);
1872	ctx->walk_control = NULL;
1873}
1874
1875/*
1876 * damos_walk_cancel() - Cancel the current DAMOS walk request.
1877 * @ctx:	The context of &damon_ctx->walk_control.
1878 *
1879 * This function is called when @ctx is deactivated by DAMOS watermarks, DAMOS
1880 * walk is requested but there is no DAMOS scheme to walk for, or the kdamond
1881 * is already out of the main loop and therefore gonna be terminated, and hence
1882 * cannot continue the walks.  This function therefore marks the walk request
1883 * as canceled, so that damos_walk() can wake up and return.
1884 */
1885static void damos_walk_cancel(struct damon_ctx *ctx)
1886{
1887	struct damos_walk_control *control;
1888
1889	mutex_lock(&ctx->walk_control_lock);
1890	control = ctx->walk_control;
1891	mutex_unlock(&ctx->walk_control_lock);
1892
1893	if (!control)
1894		return;
1895	control->canceled = true;
1896	complete(&control->completion);
1897	mutex_lock(&ctx->walk_control_lock);
1898	ctx->walk_control = NULL;
1899	mutex_unlock(&ctx->walk_control_lock);
1900}
1901
1902static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t,
1903		struct damon_region *r, struct damos *s)
1904{
1905	struct damos_quota *quota = &s->quota;
1906	unsigned long sz = damon_sz_region(r);
1907	struct timespec64 begin, end;
1908	unsigned long sz_applied = 0;
1909	unsigned long sz_ops_filter_passed = 0;
1910	/*
1911	 * We plan to support multiple context per kdamond, as DAMON sysfs
1912	 * implies with 'nr_contexts' file.  Nevertheless, only single context
1913	 * per kdamond is supported for now.  So, we can simply use '0' context
1914	 * index here.
1915	 */
1916	unsigned int cidx = 0;
1917	struct damos *siter;		/* schemes iterator */
1918	unsigned int sidx = 0;
1919	struct damon_target *titer;	/* targets iterator */
1920	unsigned int tidx = 0;
1921	bool do_trace = false;
1922
1923	/* get indices for trace_damos_before_apply() */
1924	if (trace_damos_before_apply_enabled()) {
1925		damon_for_each_scheme(siter, c) {
1926			if (siter == s)
1927				break;
1928			sidx++;
1929		}
1930		damon_for_each_target(titer, c) {
1931			if (titer == t)
1932				break;
1933			tidx++;
1934		}
1935		do_trace = true;
1936	}
1937
1938	if (c->ops.apply_scheme) {
1939		if (quota->esz && quota->charged_sz + sz > quota->esz) {
1940			sz = ALIGN_DOWN(quota->esz - quota->charged_sz,
1941					c->min_region_sz);
1942			if (!sz)
1943				goto update_stat;
1944			damon_split_region_at(t, r, sz);
1945		}
1946		if (damos_filter_out(c, t, r, s))
1947			return;
1948		ktime_get_coarse_ts64(&begin);
1949		trace_damos_before_apply(cidx, sidx, tidx, r,
1950				damon_nr_regions(t), do_trace);
1951		sz_applied = c->ops.apply_scheme(c, t, r, s,
1952				&sz_ops_filter_passed);
1953		damos_walk_call_walk(c, t, r, s, sz_ops_filter_passed);
1954		ktime_get_coarse_ts64(&end);
1955		quota->total_charged_ns += timespec64_to_ns(&end) -
1956			timespec64_to_ns(&begin);
1957		quota->charged_sz += sz;
1958		if (quota->esz && quota->charged_sz >= quota->esz) {
1959			quota->charge_target_from = t;
1960			quota->charge_addr_from = r->ar.end + 1;
1961		}
1962	}
1963	if (s->action != DAMOS_STAT)
1964		r->age = 0;
1965
1966update_stat:
1967	damos_update_stat(s, sz, sz_applied, sz_ops_filter_passed);
1968}
1969
1970static void damon_do_apply_schemes(struct damon_ctx *c,
1971				   struct damon_target *t,
1972				   struct damon_region *r)
1973{
1974	struct damos *s;
1975
1976	damon_for_each_scheme(s, c) {
1977		struct damos_quota *quota = &s->quota;
1978
1979		if (c->passed_sample_intervals < s->next_apply_sis)
1980			continue;
1981
1982		if (!s->wmarks.activated)
1983			continue;
1984
1985		/* Check the quota */
1986		if (quota->esz && quota->charged_sz >= quota->esz)
1987			continue;
1988
1989		if (damos_skip_charged_region(t, &r, s, c->min_region_sz))
1990			continue;
1991
1992		if (!damos_valid_target(c, t, r, s))
1993			continue;
1994
1995		damos_apply_scheme(c, t, r, s);
1996	}
1997}
1998
1999/*
2000 * damon_feed_loop_next_input() - get next input to achieve a target score.
2001 * @last_input	The last input.
2002 * @score	Current score that made with @last_input.
2003 *
2004 * Calculate next input to achieve the target score, based on the last input
2005 * and current score.  Assuming the input and the score are positively
2006 * proportional, calculate how much compensation should be added to or
2007 * subtracted from the last input as a proportion of the last input.  Avoid
2008 * next input always being zero by setting it non-zero always.  In short form
2009 * (assuming support of float and signed calculations), the algorithm is as
2010 * below.
2011 *
2012 * next_input = max(last_input * ((goal - current) / goal + 1), 1)
2013 *
2014 * For simple implementation, we assume the target score is always 10,000.  The
2015 * caller should adjust @score for this.
2016 *
2017 * Returns next input that assumed to achieve the target score.
2018 */
2019static unsigned long damon_feed_loop_next_input(unsigned long last_input,
2020		unsigned long score)
2021{
2022	const unsigned long goal = 10000;
2023	/* Set minimum input as 10000 to avoid compensation be zero */
2024	const unsigned long min_input = 10000;
2025	unsigned long score_goal_diff, compensation;
2026	bool over_achieving = score > goal;
2027
2028	if (score == goal)
2029		return last_input;
2030	if (score >= goal * 2)
2031		return min_input;
2032
2033	if (over_achieving)
2034		score_goal_diff = score - goal;
2035	else
2036		score_goal_diff = goal - score;
2037
2038	if (last_input < ULONG_MAX / score_goal_diff)
2039		compensation = last_input * score_goal_diff / goal;
2040	else
2041		compensation = last_input / goal * score_goal_diff;
2042
2043	if (over_achieving)
2044		return max(last_input - compensation, min_input);
2045	if (last_input < ULONG_MAX - compensation)
2046		return last_input + compensation;
2047	return ULONG_MAX;
2048}
2049
2050#ifdef CONFIG_PSI
2051
2052static u64 damos_get_some_mem_psi_total(void)
2053{
2054	if (static_branch_likely(&psi_disabled))
2055		return 0;
2056	return div_u64(psi_system.total[PSI_AVGS][PSI_MEM * 2],
2057			NSEC_PER_USEC);
2058}
2059
2060#else	/* CONFIG_PSI */
2061
2062static inline u64 damos_get_some_mem_psi_total(void)
2063{
2064	return 0;
2065};
2066
2067#endif	/* CONFIG_PSI */
2068
2069#ifdef CONFIG_NUMA
2070static __kernel_ulong_t damos_get_node_mem_bp(
2071		struct damos_quota_goal *goal)
2072{
2073	struct sysinfo i;
2074	__kernel_ulong_t numerator;
2075
2076	si_meminfo_node(&i, goal->nid);
2077	if (goal->metric == DAMOS_QUOTA_NODE_MEM_USED_BP)
2078		numerator = i.totalram - i.freeram;
2079	else	/* DAMOS_QUOTA_NODE_MEM_FREE_BP */
2080		numerator = i.freeram;
2081	return numerator * 10000 / i.totalram;
2082}
2083
2084static unsigned long damos_get_node_memcg_used_bp(
2085		struct damos_quota_goal *goal)
2086{
2087	struct mem_cgroup *memcg;
2088	struct lruvec *lruvec;
2089	unsigned long used_pages, numerator;
2090	struct sysinfo i;
2091
2092	rcu_read_lock();
2093	memcg = mem_cgroup_from_id(goal->memcg_id);
2094	if (!memcg || !mem_cgroup_tryget(memcg)) {
2095		rcu_read_unlock();
2096		if (goal->metric == DAMOS_QUOTA_NODE_MEMCG_USED_BP)
2097			return 0;
2098		else	/* DAMOS_QUOTA_NODE_MEMCG_FREE_BP */
2099			return 10000;
2100	}
2101	rcu_read_unlock();
2102
2103	mem_cgroup_flush_stats(memcg);
2104	lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(goal->nid));
2105	used_pages = lruvec_page_state(lruvec, NR_ACTIVE_ANON);
2106	used_pages += lruvec_page_state(lruvec, NR_INACTIVE_ANON);
2107	used_pages += lruvec_page_state(lruvec, NR_ACTIVE_FILE);
2108	used_pages += lruvec_page_state(lruvec, NR_INACTIVE_FILE);
2109
2110	mem_cgroup_put(memcg);
2111
2112	si_meminfo_node(&i, goal->nid);
2113	if (goal->metric == DAMOS_QUOTA_NODE_MEMCG_USED_BP)
2114		numerator = used_pages;
2115	else	/* DAMOS_QUOTA_NODE_MEMCG_FREE_BP */
2116		numerator = i.totalram - used_pages;
2117	return numerator * 10000 / i.totalram;
2118}
2119#else
2120static __kernel_ulong_t damos_get_node_mem_bp(
2121		struct damos_quota_goal *goal)
2122{
2123	return 0;
2124}
2125
2126static unsigned long damos_get_node_memcg_used_bp(
2127		struct damos_quota_goal *goal)
2128{
2129	return 0;
2130}
2131#endif
2132
2133
2134static void damos_set_quota_goal_current_value(struct damos_quota_goal *goal)
2135{
2136	u64 now_psi_total;
2137
2138	switch (goal->metric) {
2139	case DAMOS_QUOTA_USER_INPUT:
2140		/* User should already set goal->current_value */
2141		break;
2142	case DAMOS_QUOTA_SOME_MEM_PSI_US:
2143		now_psi_total = damos_get_some_mem_psi_total();
2144		goal->current_value = now_psi_total - goal->last_psi_total;
2145		goal->last_psi_total = now_psi_total;
2146		break;
2147	case DAMOS_QUOTA_NODE_MEM_USED_BP:
2148	case DAMOS_QUOTA_NODE_MEM_FREE_BP:
2149		goal->current_value = damos_get_node_mem_bp(goal);
2150		break;
2151	case DAMOS_QUOTA_NODE_MEMCG_USED_BP:
2152	case DAMOS_QUOTA_NODE_MEMCG_FREE_BP:
2153		goal->current_value = damos_get_node_memcg_used_bp(goal);
2154		break;
2155	default:
2156		break;
2157	}
2158}
2159
2160/* Return the highest score since it makes schemes least aggressive */
2161static unsigned long damos_quota_score(struct damos_quota *quota)
2162{
2163	struct damos_quota_goal *goal;
2164	unsigned long highest_score = 0;
2165
2166	damos_for_each_quota_goal(goal, quota) {
2167		damos_set_quota_goal_current_value(goal);
2168		highest_score = max(highest_score,
2169				goal->current_value * 10000 /
2170				goal->target_value);
2171	}
2172
2173	return highest_score;
2174}
2175
2176/*
2177 * Called only if quota->ms, or quota->sz are set, or quota->goals is not empty
2178 */
2179static void damos_set_effective_quota(struct damos_quota *quota)
2180{
2181	unsigned long throughput;
2182	unsigned long esz = ULONG_MAX;
2183
2184	if (!quota->ms && list_empty(&quota->goals)) {
2185		quota->esz = quota->sz;
2186		return;
2187	}
2188
2189	if (!list_empty(&quota->goals)) {
2190		unsigned long score = damos_quota_score(quota);
2191
2192		quota->esz_bp = damon_feed_loop_next_input(
2193				max(quota->esz_bp, 10000UL),
2194				score);
2195		esz = quota->esz_bp / 10000;
2196	}
2197
2198	if (quota->ms) {
2199		if (quota->total_charged_ns)
2200			throughput = mult_frac(quota->total_charged_sz, 1000000,
2201							quota->total_charged_ns);
2202		else
2203			throughput = PAGE_SIZE * 1024;
2204		esz = min(throughput * quota->ms, esz);
2205	}
2206
2207	if (quota->sz && quota->sz < esz)
2208		esz = quota->sz;
2209
2210	quota->esz = esz;
2211}
2212
2213static void damos_trace_esz(struct damon_ctx *c, struct damos *s,
2214		struct damos_quota *quota)
2215{
2216	unsigned int cidx = 0, sidx = 0;
2217	struct damos *siter;
2218
2219	damon_for_each_scheme(siter, c) {
2220		if (siter == s)
2221			break;
2222		sidx++;
2223	}
2224	trace_damos_esz(cidx, sidx, quota->esz);
2225}
2226
2227static void damos_adjust_quota(struct damon_ctx *c, struct damos *s)
2228{
2229	struct damos_quota *quota = &s->quota;
2230	struct damon_target *t;
2231	struct damon_region *r;
2232	unsigned long cumulated_sz, cached_esz;
2233	unsigned int score, max_score = 0;
2234
2235	if (!quota->ms && !quota->sz && list_empty(&quota->goals))
2236		return;
2237
2238	/* First charge window */
2239	if (!quota->total_charged_sz && !quota->charged_from) {
2240		quota->charged_from = jiffies;
2241		damos_set_effective_quota(quota);
2242	}
2243
2244	/* New charge window starts */
2245	if (time_after_eq(jiffies, quota->charged_from +
2246				msecs_to_jiffies(quota->reset_interval))) {
2247		if (quota->esz && quota->charged_sz >= quota->esz)
2248			s->stat.qt_exceeds++;
2249		quota->total_charged_sz += quota->charged_sz;
2250		quota->charged_from = jiffies;
2251		quota->charged_sz = 0;
2252		if (trace_damos_esz_enabled())
2253			cached_esz = quota->esz;
2254		damos_set_effective_quota(quota);
2255		if (trace_damos_esz_enabled() && quota->esz != cached_esz)
2256			damos_trace_esz(c, s, quota);
2257	}
2258
2259	if (!c->ops.get_scheme_score)
2260		return;
2261
2262	/* Fill up the score histogram */
2263	memset(c->regions_score_histogram, 0,
2264			sizeof(*c->regions_score_histogram) *
2265			(DAMOS_MAX_SCORE + 1));
2266	damon_for_each_target(t, c) {
2267		damon_for_each_region(r, t) {
2268			if (!__damos_valid_target(r, s))
2269				continue;
2270			score = c->ops.get_scheme_score(c, t, r, s);
2271			c->regions_score_histogram[score] +=
2272				damon_sz_region(r);
2273			if (score > max_score)
2274				max_score = score;
2275		}
2276	}
2277
2278	/* Set the min score limit */
2279	for (cumulated_sz = 0, score = max_score; ; score--) {
2280		cumulated_sz += c->regions_score_histogram[score];
2281		if (cumulated_sz >= quota->esz || !score)
2282			break;
2283	}
2284	quota->min_score = score;
2285}
2286
2287static void kdamond_apply_schemes(struct damon_ctx *c)
2288{
2289	struct damon_target *t;
2290	struct damon_region *r, *next_r;
2291	struct damos *s;
2292	unsigned long sample_interval = c->attrs.sample_interval ?
2293		c->attrs.sample_interval : 1;
2294	bool has_schemes_to_apply = false;
2295
2296	damon_for_each_scheme(s, c) {
2297		if (c->passed_sample_intervals < s->next_apply_sis)
2298			continue;
2299
2300		if (!s->wmarks.activated)
2301			continue;
2302
2303		has_schemes_to_apply = true;
2304
2305		damos_adjust_quota(c, s);
2306	}
2307
2308	if (!has_schemes_to_apply)
2309		return;
2310
2311	mutex_lock(&c->walk_control_lock);
2312	damon_for_each_target(t, c) {
2313		damon_for_each_region_safe(r, next_r, t)
2314			damon_do_apply_schemes(c, t, r);
2315	}
2316
2317	damon_for_each_scheme(s, c) {
2318		if (c->passed_sample_intervals < s->next_apply_sis)
2319			continue;
2320		damos_walk_complete(c, s);
2321		s->next_apply_sis = c->passed_sample_intervals +
2322			(s->apply_interval_us ? s->apply_interval_us :
2323			 c->attrs.aggr_interval) / sample_interval;
2324		s->last_applied = NULL;
2325	}
2326	mutex_unlock(&c->walk_control_lock);
2327}
2328
2329/*
2330 * Merge two adjacent regions into one region
2331 */
2332static void damon_merge_two_regions(struct damon_target *t,
2333		struct damon_region *l, struct damon_region *r)
2334{
2335	unsigned long sz_l = damon_sz_region(l), sz_r = damon_sz_region(r);
2336
2337	l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) /
2338			(sz_l + sz_r);
2339	l->nr_accesses_bp = l->nr_accesses * 10000;
2340	l->age = (l->age * sz_l + r->age * sz_r) / (sz_l + sz_r);
2341	l->ar.end = r->ar.end;
2342	damon_destroy_region(r, t);
2343}
2344
2345/*
2346 * Merge adjacent regions having similar access frequencies
2347 *
2348 * t		target affected by this merge operation
2349 * thres	'->nr_accesses' diff threshold for the merge
2350 * sz_limit	size upper limit of each region
2351 */
2352static void damon_merge_regions_of(struct damon_target *t, unsigned int thres,
2353				   unsigned long sz_limit)
2354{
2355	struct damon_region *r, *prev = NULL, *next;
2356
2357	damon_for_each_region_safe(r, next, t) {
2358		if (abs(r->nr_accesses - r->last_nr_accesses) > thres)
2359			r->age = 0;
2360		else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0))
2361			r->age = 0;
2362		else
2363			r->age++;
2364
2365		if (prev && prev->ar.end == r->ar.start &&
2366		    abs(prev->nr_accesses - r->nr_accesses) <= thres &&
2367		    damon_sz_region(prev) + damon_sz_region(r) <= sz_limit)
2368			damon_merge_two_regions(t, prev, r);
2369		else
2370			prev = r;
2371	}
2372}
2373
2374/*
2375 * Merge adjacent regions having similar access frequencies
2376 *
2377 * threshold	'->nr_accesses' diff threshold for the merge
2378 * sz_limit	size upper limit of each region
2379 *
2380 * This function merges monitoring target regions which are adjacent and their
2381 * access frequencies are similar.  This is for minimizing the monitoring
2382 * overhead under the dynamically changeable access pattern.  If a merge was
2383 * unnecessarily made, later 'kdamond_split_regions()' will revert it.
2384 *
2385 * The total number of regions could be higher than the user-defined limit,
2386 * max_nr_regions for some cases.  For example, the user can update
2387 * max_nr_regions to a number that lower than the current number of regions
2388 * while DAMON is running.  For such a case, repeat merging until the limit is
2389 * met while increasing @threshold up to possible maximum level.
2390 */
2391static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold,
2392				  unsigned long sz_limit)
2393{
2394	struct damon_target *t;
2395	unsigned int nr_regions;
2396	unsigned int max_thres;
2397
2398	max_thres = c->attrs.aggr_interval /
2399		(c->attrs.sample_interval ?  c->attrs.sample_interval : 1);
2400	do {
2401		nr_regions = 0;
2402		damon_for_each_target(t, c) {
2403			damon_merge_regions_of(t, threshold, sz_limit);
2404			nr_regions += damon_nr_regions(t);
2405		}
2406		threshold = max(1, threshold * 2);
2407	} while (nr_regions > c->attrs.max_nr_regions &&
2408			threshold / 2 < max_thres);
2409}
2410
2411/*
2412 * Split a region in two
2413 *
2414 * r		the region to be split
2415 * sz_r		size of the first sub-region that will be made
2416 */
2417static void damon_split_region_at(struct damon_target *t,
2418				  struct damon_region *r, unsigned long sz_r)
2419{
2420	struct damon_region *new;
2421
2422	new = damon_new_region(r->ar.start + sz_r, r->ar.end);
2423	if (!new)
2424		return;
2425
2426	r->ar.end = new->ar.start;
2427
2428	new->age = r->age;
2429	new->last_nr_accesses = r->last_nr_accesses;
2430	new->nr_accesses_bp = r->nr_accesses_bp;
2431	new->nr_accesses = r->nr_accesses;
2432
2433	damon_insert_region(new, r, damon_next_region(r), t);
2434}
2435
2436/* Split every region in the given target into 'nr_subs' regions */
2437static void damon_split_regions_of(struct damon_target *t, int nr_subs,
2438				  unsigned long min_region_sz)
2439{
2440	struct damon_region *r, *next;
2441	unsigned long sz_region, sz_sub = 0;
2442	int i;
2443
2444	damon_for_each_region_safe(r, next, t) {
2445		sz_region = damon_sz_region(r);
2446
2447		for (i = 0; i < nr_subs - 1 &&
2448				sz_region > 2 * min_region_sz; i++) {
2449			/*
2450			 * Randomly select size of left sub-region to be at
2451			 * least 10 percent and at most 90% of original region
2452			 */
2453			sz_sub = ALIGN_DOWN(damon_rand(1, 10) *
2454					sz_region / 10, min_region_sz);
2455			/* Do not allow blank region */
2456			if (sz_sub == 0 || sz_sub >= sz_region)
2457				continue;
2458
2459			damon_split_region_at(t, r, sz_sub);
2460			sz_region = sz_sub;
2461		}
2462	}
2463}
2464
2465/*
2466 * Split every target region into randomly-sized small regions
2467 *
2468 * This function splits every target region into random-sized small regions if
2469 * current total number of the regions is equal or smaller than half of the
2470 * user-specified maximum number of regions.  This is for maximizing the
2471 * monitoring accuracy under the dynamically changeable access patterns.  If a
2472 * split was unnecessarily made, later 'kdamond_merge_regions()' will revert
2473 * it.
2474 */
2475static void kdamond_split_regions(struct damon_ctx *ctx)
2476{
2477	struct damon_target *t;
2478	unsigned int nr_regions = 0;
2479	static unsigned int last_nr_regions;
2480	int nr_subregions = 2;
2481
2482	damon_for_each_target(t, ctx)
2483		nr_regions += damon_nr_regions(t);
2484
2485	if (nr_regions > ctx->attrs.max_nr_regions / 2)
2486		return;
2487
2488	/* Maybe the middle of the region has different access frequency */
2489	if (last_nr_regions == nr_regions &&
2490			nr_regions < ctx->attrs.max_nr_regions / 3)
2491		nr_subregions = 3;
2492
2493	damon_for_each_target(t, ctx)
2494		damon_split_regions_of(t, nr_subregions, ctx->min_region_sz);
2495
2496	last_nr_regions = nr_regions;
2497}
2498
2499/*
2500 * Check whether current monitoring should be stopped
2501 *
2502 * The monitoring is stopped when either the user requested to stop, or all
2503 * monitoring targets are invalid.
2504 *
2505 * Returns true if need to stop current monitoring.
2506 */
2507static bool kdamond_need_stop(struct damon_ctx *ctx)
2508{
2509	struct damon_target *t;
2510
2511	if (kthread_should_stop())
2512		return true;
2513
2514	if (!ctx->ops.target_valid)
2515		return false;
2516
2517	damon_for_each_target(t, ctx) {
2518		if (ctx->ops.target_valid(t))
2519			return false;
2520	}
2521
2522	return true;
2523}
2524
2525static int damos_get_wmark_metric_value(enum damos_wmark_metric metric,
2526					unsigned long *metric_value)
2527{
2528	switch (metric) {
2529	case DAMOS_WMARK_FREE_MEM_RATE:
2530		*metric_value = global_zone_page_state(NR_FREE_PAGES) * 1000 /
2531		       totalram_pages();
2532		return 0;
2533	default:
2534		break;
2535	}
2536	return -EINVAL;
2537}
2538
2539/*
2540 * Returns zero if the scheme is active.  Else, returns time to wait for next
2541 * watermark check in micro-seconds.
2542 */
2543static unsigned long damos_wmark_wait_us(struct damos *scheme)
2544{
2545	unsigned long metric;
2546
2547	if (damos_get_wmark_metric_value(scheme->wmarks.metric, &metric))
2548		return 0;
2549
2550	/* higher than high watermark or lower than low watermark */
2551	if (metric > scheme->wmarks.high || scheme->wmarks.low > metric) {
2552		if (scheme->wmarks.activated)
2553			pr_debug("deactivate a scheme (%d) for %s wmark\n",
2554				 scheme->action,
2555				 str_high_low(metric > scheme->wmarks.high));
2556		scheme->wmarks.activated = false;
2557		return scheme->wmarks.interval;
2558	}
2559
2560	/* inactive and higher than middle watermark */
2561	if ((scheme->wmarks.high >= metric && metric >= scheme->wmarks.mid) &&
2562			!scheme->wmarks.activated)
2563		return scheme->wmarks.interval;
2564
2565	if (!scheme->wmarks.activated)
2566		pr_debug("activate a scheme (%d)\n", scheme->action);
2567	scheme->wmarks.activated = true;
2568	return 0;
2569}
2570
2571static void kdamond_usleep(unsigned long usecs)
2572{
2573	if (usecs >= USLEEP_RANGE_UPPER_BOUND)
2574		schedule_timeout_idle(usecs_to_jiffies(usecs));
2575	else
2576		usleep_range_idle(usecs, usecs + 1);
2577}
2578
2579/*
2580 * kdamond_call() - handle damon_call_control objects.
2581 * @ctx:	The &struct damon_ctx of the kdamond.
2582 * @cancel:	Whether to cancel the invocation of the function.
2583 *
2584 * If there are &struct damon_call_control requests that registered via
2585 * &damon_call() on @ctx, do or cancel the invocation of the function depending
2586 * on @cancel.  @cancel is set when the kdamond is already out of the main loop
2587 * and therefore will be terminated.
2588 */
2589static void kdamond_call(struct damon_ctx *ctx, bool cancel)
2590{
2591	struct damon_call_control *control;
2592	LIST_HEAD(repeat_controls);
2593	int ret = 0;
2594
2595	while (true) {
2596		mutex_lock(&ctx->call_controls_lock);
2597		control = list_first_entry_or_null(&ctx->call_controls,
2598				struct damon_call_control, list);
2599		mutex_unlock(&ctx->call_controls_lock);
2600		if (!control)
2601			break;
2602		if (cancel) {
2603			control->canceled = true;
2604		} else {
2605			ret = control->fn(control->data);
2606			control->return_code = ret;
2607		}
2608		mutex_lock(&ctx->call_controls_lock);
2609		list_del(&control->list);
2610		mutex_unlock(&ctx->call_controls_lock);
2611		if (!control->repeat) {
2612			complete(&control->completion);
2613		} else if (control->canceled && control->dealloc_on_cancel) {
2614			kfree(control);
2615		} else {
2616			list_add(&control->list, &repeat_controls);
2617		}
2618		if (!cancel && ctx->maybe_corrupted)
2619			break;
2620	}
2621	control = list_first_entry_or_null(&repeat_controls,
2622			struct damon_call_control, list);
2623	if (!control || cancel)
2624		return;
2625	mutex_lock(&ctx->call_controls_lock);
2626	list_add_tail(&control->list, &ctx->call_controls);
2627	mutex_unlock(&ctx->call_controls_lock);
2628}
2629
2630/* Returns negative error code if it's not activated but should return */
2631static int kdamond_wait_activation(struct damon_ctx *ctx)
2632{
2633	struct damos *s;
2634	unsigned long wait_time;
2635	unsigned long min_wait_time = 0;
2636	bool init_wait_time = false;
2637
2638	while (!kdamond_need_stop(ctx)) {
2639		damon_for_each_scheme(s, ctx) {
2640			wait_time = damos_wmark_wait_us(s);
2641			if (!init_wait_time || wait_time < min_wait_time) {
2642				init_wait_time = true;
2643				min_wait_time = wait_time;
2644			}
2645		}
2646		if (!min_wait_time)
2647			return 0;
2648
2649		kdamond_usleep(min_wait_time);
2650
2651		kdamond_call(ctx, false);
2652		if (ctx->maybe_corrupted)
2653			return -EINVAL;
2654		damos_walk_cancel(ctx);
2655	}
2656	return -EBUSY;
2657}
2658
2659static void kdamond_init_ctx(struct damon_ctx *ctx)
2660{
2661	unsigned long sample_interval = ctx->attrs.sample_interval ?
2662		ctx->attrs.sample_interval : 1;
2663	unsigned long apply_interval;
2664	struct damos *scheme;
2665
2666	ctx->passed_sample_intervals = 0;
2667	ctx->next_aggregation_sis = ctx->attrs.aggr_interval / sample_interval;
2668	ctx->next_ops_update_sis = ctx->attrs.ops_update_interval /
2669		sample_interval;
2670	ctx->next_intervals_tune_sis = ctx->next_aggregation_sis *
2671		ctx->attrs.intervals_goal.aggrs;
2672
2673	damon_for_each_scheme(scheme, ctx) {
2674		apply_interval = scheme->apply_interval_us ?
2675			scheme->apply_interval_us : ctx->attrs.aggr_interval;
2676		scheme->next_apply_sis = apply_interval / sample_interval;
2677		damos_set_filters_default_reject(scheme);
2678	}
2679}
2680
2681/*
2682 * The monitoring daemon that runs as a kernel thread
2683 */
2684static int kdamond_fn(void *data)
2685{
2686	struct damon_ctx *ctx = data;
2687	struct damon_target *t;
2688	struct damon_region *r, *next;
2689	unsigned int max_nr_accesses = 0;
2690	unsigned long sz_limit = 0;
2691
2692	pr_debug("kdamond (%d) starts\n", current->pid);
2693
2694	complete(&ctx->kdamond_started);
2695	kdamond_init_ctx(ctx);
2696
2697	if (ctx->ops.init)
2698		ctx->ops.init(ctx);
2699	ctx->regions_score_histogram = kmalloc_array(DAMOS_MAX_SCORE + 1,
2700			sizeof(*ctx->regions_score_histogram), GFP_KERNEL);
2701	if (!ctx->regions_score_histogram)
2702		goto done;
2703
2704	sz_limit = damon_region_sz_limit(ctx);
2705
2706	while (!kdamond_need_stop(ctx)) {
2707		/*
2708		 * ctx->attrs and ctx->next_{aggregation,ops_update}_sis could
2709		 * be changed from kdamond_call().  Read the values here, and
2710		 * use those for this iteration.  That is, damon_set_attrs()
2711		 * updated new values are respected from next iteration.
2712		 */
2713		unsigned long next_aggregation_sis = ctx->next_aggregation_sis;
2714		unsigned long next_ops_update_sis = ctx->next_ops_update_sis;
2715		unsigned long sample_interval = ctx->attrs.sample_interval;
2716
2717		if (kdamond_wait_activation(ctx))
2718			break;
2719
2720		if (ctx->ops.prepare_access_checks)
2721			ctx->ops.prepare_access_checks(ctx);
2722
2723		kdamond_usleep(sample_interval);
2724		ctx->passed_sample_intervals++;
2725
2726		if (ctx->ops.check_accesses)
2727			max_nr_accesses = ctx->ops.check_accesses(ctx);
2728
2729		if (ctx->passed_sample_intervals >= next_aggregation_sis)
2730			kdamond_merge_regions(ctx,
2731					max_nr_accesses / 10,
2732					sz_limit);
2733
2734		/*
2735		 * do kdamond_call() and kdamond_apply_schemes() after
2736		 * kdamond_merge_regions() if possible, to reduce overhead
2737		 */
2738		kdamond_call(ctx, false);
2739		if (ctx->maybe_corrupted)
2740			break;
2741		if (!list_empty(&ctx->schemes))
2742			kdamond_apply_schemes(ctx);
2743		else
2744			damos_walk_cancel(ctx);
2745
2746		sample_interval = ctx->attrs.sample_interval ?
2747			ctx->attrs.sample_interval : 1;
2748		if (ctx->passed_sample_intervals >= next_aggregation_sis) {
2749			if (ctx->attrs.intervals_goal.aggrs &&
2750					ctx->passed_sample_intervals >=
2751					ctx->next_intervals_tune_sis) {
2752				/*
2753				 * ctx->next_aggregation_sis might be updated
2754				 * from kdamond_call().  In the case,
2755				 * damon_set_attrs() which will be called from
2756				 * kdamond_tune_interval() may wrongly think
2757				 * this is in the middle of the current
2758				 * aggregation, and make aggregation
2759				 * information reset for all regions.  Then,
2760				 * following kdamond_reset_aggregated() call
2761				 * will make the region information invalid,
2762				 * particularly for ->nr_accesses_bp.
2763				 *
2764				 * Reset ->next_aggregation_sis to avoid that.
2765				 * It will anyway correctly updated after this
2766				 * if caluse.
2767				 */
2768				ctx->next_aggregation_sis =
2769					next_aggregation_sis;
2770				ctx->next_intervals_tune_sis +=
2771					ctx->attrs.aggr_samples *
2772					ctx->attrs.intervals_goal.aggrs;
2773				kdamond_tune_intervals(ctx);
2774				sample_interval = ctx->attrs.sample_interval ?
2775					ctx->attrs.sample_interval : 1;
2776
2777			}
2778			ctx->next_aggregation_sis = next_aggregation_sis +
2779				ctx->attrs.aggr_interval / sample_interval;
2780
2781			kdamond_reset_aggregated(ctx);
2782			kdamond_split_regions(ctx);
2783		}
2784
2785		if (ctx->passed_sample_intervals >= next_ops_update_sis) {
2786			ctx->next_ops_update_sis = next_ops_update_sis +
2787				ctx->attrs.ops_update_interval /
2788				sample_interval;
2789			if (ctx->ops.update)
2790				ctx->ops.update(ctx);
2791			sz_limit = damon_region_sz_limit(ctx);
2792		}
2793	}
2794done:
2795	damon_for_each_target(t, ctx) {
2796		damon_for_each_region_safe(r, next, t)
2797			damon_destroy_region(r, t);
2798	}
2799
2800	if (ctx->ops.cleanup)
2801		ctx->ops.cleanup(ctx);
2802	kfree(ctx->regions_score_histogram);
2803	kdamond_call(ctx, true);
2804
2805	pr_debug("kdamond (%d) finishes\n", current->pid);
2806	mutex_lock(&ctx->kdamond_lock);
2807	ctx->kdamond = NULL;
2808	mutex_unlock(&ctx->kdamond_lock);
2809
2810	damos_walk_cancel(ctx);
2811
2812	mutex_lock(&damon_lock);
2813	nr_running_ctxs--;
2814	if (!nr_running_ctxs && running_exclusive_ctxs)
2815		running_exclusive_ctxs = false;
2816	mutex_unlock(&damon_lock);
2817
2818	damon_destroy_targets(ctx);
2819	return 0;
2820}
2821
2822/*
2823 * struct damon_system_ram_region - System RAM resource address region of
2824 *				    [@start, @end).
2825 * @start:	Start address of the region (inclusive).
2826 * @end:	End address of the region (exclusive).
2827 */
2828struct damon_system_ram_region {
2829	unsigned long start;
2830	unsigned long end;
2831};
2832
2833static int walk_system_ram(struct resource *res, void *arg)
2834{
2835	struct damon_system_ram_region *a = arg;
2836
2837	if (a->end - a->start < resource_size(res)) {
2838		a->start = res->start;
2839		a->end = res->end;
2840	}
2841	return 0;
2842}
2843
2844/*
2845 * Find biggest 'System RAM' resource and store its start and end address in
2846 * @start and @end, respectively.  If no System RAM is found, returns false.
2847 */
2848static bool damon_find_biggest_system_ram(unsigned long *start,
2849						unsigned long *end)
2850
2851{
2852	struct damon_system_ram_region arg = {};
2853
2854	walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram);
2855	if (arg.end <= arg.start)
2856		return false;
2857
2858	*start = arg.start;
2859	*end = arg.end;
2860	return true;
2861}
2862
2863/**
2864 * damon_set_region_biggest_system_ram_default() - Set the region of the given
2865 * monitoring target as requested, or biggest 'System RAM'.
2866 * @t:		The monitoring target to set the region.
2867 * @start:	The pointer to the start address of the region.
2868 * @end:	The pointer to the end address of the region.
2869 * @min_region_sz:	Minimum region size.
2870 *
2871 * This function sets the region of @t as requested by @start and @end.  If the
2872 * values of @start and @end are zero, however, this function finds the biggest
2873 * 'System RAM' resource and sets the region to cover the resource.  In the
2874 * latter case, this function saves the start and end addresses of the resource
2875 * in @start and @end, respectively.
2876 *
2877 * Return: 0 on success, negative error code otherwise.
2878 */
2879int damon_set_region_biggest_system_ram_default(struct damon_target *t,
2880			unsigned long *start, unsigned long *end,
2881			unsigned long min_region_sz)
2882{
2883	struct damon_addr_range addr_range;
2884
2885	if (*start > *end)
2886		return -EINVAL;
2887
2888	if (!*start && !*end &&
2889		!damon_find_biggest_system_ram(start, end))
2890		return -EINVAL;
2891
2892	addr_range.start = *start;
2893	addr_range.end = *end;
2894	return damon_set_regions(t, &addr_range, 1, min_region_sz);
2895}
2896
2897/*
2898 * damon_moving_sum() - Calculate an inferred moving sum value.
2899 * @mvsum:	Inferred sum of the last @len_window values.
2900 * @nomvsum:	Non-moving sum of the last discrete @len_window window values.
2901 * @len_window:	The number of last values to take care of.
2902 * @new_value:	New value that will be added to the pseudo moving sum.
2903 *
2904 * Moving sum (moving average * window size) is good for handling noise, but
2905 * the cost of keeping past values can be high for arbitrary window size.  This
2906 * function implements a lightweight pseudo moving sum function that doesn't
2907 * keep the past window values.
2908 *
2909 * It simply assumes there was no noise in the past, and get the no-noise
2910 * assumed past value to drop from @nomvsum and @len_window.  @nomvsum is a
2911 * non-moving sum of the last window.  For example, if @len_window is 10 and we
2912 * have 25 values, @nomvsum is the sum of the 11th to 20th values of the 25
2913 * values.  Hence, this function simply drops @nomvsum / @len_window from
2914 * given @mvsum and add @new_value.
2915 *
2916 * For example, if @len_window is 10 and @nomvsum is 50, the last 10 values for
2917 * the last window could be vary, e.g., 0, 10, 0, 10, 0, 10, 0, 0, 0, 20.  For
2918 * calculating next moving sum with a new value, we should drop 0 from 50 and
2919 * add the new value.  However, this function assumes it got value 5 for each
2920 * of the last ten times.  Based on the assumption, when the next value is
2921 * measured, it drops the assumed past value, 5 from the current sum, and add
2922 * the new value to get the updated pseduo-moving average.
2923 *
2924 * This means the value could have errors, but the errors will be disappeared
2925 * for every @len_window aligned calls.  For example, if @len_window is 10, the
2926 * pseudo moving sum with 11th value to 19th value would have an error.  But
2927 * the sum with 20th value will not have the error.
2928 *
2929 * Return: Pseudo-moving average after getting the @new_value.
2930 */
2931static unsigned int damon_moving_sum(unsigned int mvsum, unsigned int nomvsum,
2932		unsigned int len_window, unsigned int new_value)
2933{
2934	return mvsum - nomvsum / len_window + new_value;
2935}
2936
2937/**
2938 * damon_update_region_access_rate() - Update the access rate of a region.
2939 * @r:		The DAMON region to update for its access check result.
2940 * @accessed:	Whether the region has accessed during last sampling interval.
2941 * @attrs:	The damon_attrs of the DAMON context.
2942 *
2943 * Update the access rate of a region with the region's last sampling interval
2944 * access check result.
2945 *
2946 * Usually this will be called by &damon_operations->check_accesses callback.
2947 */
2948void damon_update_region_access_rate(struct damon_region *r, bool accessed,
2949		struct damon_attrs *attrs)
2950{
2951	unsigned int len_window = 1;
2952
2953	/*
2954	 * sample_interval can be zero, but cannot be larger than
2955	 * aggr_interval, owing to validation of damon_set_attrs().
2956	 */
2957	if (attrs->sample_interval)
2958		len_window = damon_max_nr_accesses(attrs);
2959	r->nr_accesses_bp = damon_moving_sum(r->nr_accesses_bp,
2960			r->last_nr_accesses * 10000, len_window,
2961			accessed ? 10000 : 0);
2962
2963	if (accessed)
2964		r->nr_accesses++;
2965}
2966
2967/**
2968 * damon_initialized() - Return if DAMON is ready to be used.
2969 *
2970 * Return: true if DAMON is ready to be used, false otherwise.
2971 */
2972bool damon_initialized(void)
2973{
2974	return damon_region_cache != NULL;
2975}
2976
2977static int __init damon_init(void)
2978{
2979	damon_region_cache = KMEM_CACHE(damon_region, 0);
2980	if (unlikely(!damon_region_cache)) {
2981		pr_err("creating damon_region_cache fails\n");
2982		return -ENOMEM;
2983	}
2984
2985	return 0;
2986}
2987
2988subsys_initcall(damon_init);
2989
2990#include "tests/core-kunit.h"