ksm_tests.c

  1// SPDX-License-Identifier: GPL-2.0
  2
  3#include <sys/mman.h>
  4#include <sys/prctl.h>
  5#include <sys/wait.h>
  6#include <stdbool.h>
  7#include <time.h>
  8#include <string.h>
  9#include <numa.h>
 10#include <unistd.h>
 11#include <fcntl.h>
 12#include <stdint.h>
 13#include <err.h>
 14
 15#include "kselftest.h"
 16#include <include/vdso/time64.h>
 17#include "vm_util.h"
 18#include "thp_settings.h"
 19
 20#define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/"
 21#define KSM_FP(s) (KSM_SYSFS_PATH s)
 22#define KSM_SCAN_LIMIT_SEC_DEFAULT 120
 23#define KSM_PAGE_COUNT_DEFAULT 10l
 24#define KSM_PROT_STR_DEFAULT "rw"
 25#define KSM_USE_ZERO_PAGES_DEFAULT false
 26#define KSM_MERGE_ACROSS_NODES_DEFAULT true
 27#define KSM_MERGE_TYPE_DEFAULT 0
 28#define MB (1ul << 20)
 29
 30struct ksm_sysfs {
 31	unsigned long max_page_sharing;
 32	unsigned long merge_across_nodes;
 33	unsigned long pages_to_scan;
 34	unsigned long run;
 35	unsigned long sleep_millisecs;
 36	unsigned long stable_node_chains_prune_millisecs;
 37	unsigned long use_zero_pages;
 38};
 39
 40enum ksm_merge_type {
 41	KSM_MERGE_MADVISE,
 42	KSM_MERGE_PRCTL,
 43	KSM_MERGE_LAST = KSM_MERGE_PRCTL
 44};
 45
 46enum ksm_test_name {
 47	CHECK_KSM_MERGE,
 48	CHECK_KSM_UNMERGE,
 49	CHECK_KSM_GET_MERGE_TYPE,
 50	CHECK_KSM_ZERO_PAGE_MERGE,
 51	CHECK_KSM_NUMA_MERGE,
 52	KSM_MERGE_TIME,
 53	KSM_MERGE_TIME_HUGE_PAGES,
 54	KSM_UNMERGE_TIME,
 55	KSM_COW_TIME
 56};
 57
 58int debug;
 59
 60static int ksm_write_sysfs(const char *file_path, unsigned long val)
 61{
 62	return write_sysfs(file_path, val);
 63}
 64
 65static int ksm_read_sysfs(const char *file_path, unsigned long *val)
 66{
 67	return read_sysfs(file_path, val);
 68}
 69
 70static void ksm_print_sysfs(void)
 71{
 72	unsigned long max_page_sharing, pages_sharing, pages_shared;
 73	unsigned long full_scans, pages_unshared, pages_volatile;
 74	unsigned long stable_node_chains, stable_node_dups;
 75	long general_profit;
 76
 77	if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||
 78	    ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||
 79	    ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing) ||
 80	    ksm_read_sysfs(KSM_FP("full_scans"), &full_scans) ||
 81	    ksm_read_sysfs(KSM_FP("pages_unshared"), &pages_unshared) ||
 82	    ksm_read_sysfs(KSM_FP("pages_volatile"), &pages_volatile) ||
 83	    ksm_read_sysfs(KSM_FP("stable_node_chains"), &stable_node_chains) ||
 84	    ksm_read_sysfs(KSM_FP("stable_node_dups"), &stable_node_dups) ||
 85	    ksm_read_sysfs(KSM_FP("general_profit"), (unsigned long *)&general_profit))
 86		return;
 87
 88	printf("pages_shared      : %lu\n", pages_shared);
 89	printf("pages_sharing     : %lu\n", pages_sharing);
 90	printf("max_page_sharing  : %lu\n", max_page_sharing);
 91	printf("full_scans        : %lu\n", full_scans);
 92	printf("pages_unshared    : %lu\n", pages_unshared);
 93	printf("pages_volatile    : %lu\n", pages_volatile);
 94	printf("stable_node_chains: %lu\n", stable_node_chains);
 95	printf("stable_node_dups  : %lu\n", stable_node_dups);
 96	printf("general_profit    : %ld\n", general_profit);
 97}
 98
 99static void ksm_print_procfs(void)
100{
101	const char *file_name = "/proc/self/ksm_stat";
102	char buffer[512];
103	FILE *f = fopen(file_name, "r");
104
105	if (!f) {
106		fprintf(stderr, "f %s\n", file_name);
107		perror("fopen");
108		return;
109	}
110
111	while (fgets(buffer, sizeof(buffer), f))
112		printf("%s", buffer);
113
114	fclose(f);
115}
116
117static int str_to_prot(char *prot_str)
118{
119	int prot = 0;
120
121	if ((strchr(prot_str, 'r')) != NULL)
122		prot |= PROT_READ;
123	if ((strchr(prot_str, 'w')) != NULL)
124		prot |= PROT_WRITE;
125	if ((strchr(prot_str, 'x')) != NULL)
126		prot |= PROT_EXEC;
127
128	return prot;
129}
130
131static void print_help(void)
132{
133	printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n"
134	       "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n");
135
136	printf("Supported <test type>:\n"
137	       " -M (page merging)\n"
138	       " -Z (zero pages merging)\n"
139	       " -N (merging of pages in different NUMA nodes)\n"
140	       " -U (page unmerging)\n"
141	       " -P evaluate merging time and speed.\n"
142	       "    For this test, the size of duplicated memory area (in MiB)\n"
143	       "    must be provided using -s option\n"
144	       " -H evaluate merging time and speed of area allocated mostly with huge pages\n"
145	       "    For this test, the size of duplicated memory area (in MiB)\n"
146	       "    must be provided using -s option\n"
147	       " -D evaluate unmerging time and speed when disabling KSM.\n"
148	       "    For this test, the size of duplicated memory area (in MiB)\n"
149	       "    must be provided using -s option\n"
150	       " -C evaluate the time required to break COW of merged pages.\n\n");
151
152	printf(" -a: specify the access protections of pages.\n"
153	       "     <prot> must be of the form [rwx].\n"
154	       "     Default: %s\n", KSM_PROT_STR_DEFAULT);
155	printf(" -p: specify the number of pages to test.\n"
156	       "     Default: %ld\n", KSM_PAGE_COUNT_DEFAULT);
157	printf(" -l: limit the maximum running time (in seconds) for a test.\n"
158	       "     Default: %d seconds\n", KSM_SCAN_LIMIT_SEC_DEFAULT);
159	printf(" -z: change use_zero_pages tunable\n"
160	       "     Default: %d\n", KSM_USE_ZERO_PAGES_DEFAULT);
161	printf(" -m: change merge_across_nodes tunable\n"
162	       "     Default: %d\n", KSM_MERGE_ACROSS_NODES_DEFAULT);
163	printf(" -d: turn debugging output on\n");
164	printf(" -s: the size of duplicated memory area (in MiB)\n");
165	printf(" -t: KSM merge type\n"
166	       "     Default: 0\n"
167	       "     0: madvise merging\n"
168	       "     1: prctl merging\n");
169
170	exit(0);
171}
172
173static void  *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size)
174{
175	void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0);
176
177	if (!map_ptr) {
178		perror("mmap");
179		return NULL;
180	}
181	memset(map_ptr, data, map_size);
182	if (mprotect(map_ptr, map_size, prot)) {
183		perror("mprotect");
184		munmap(map_ptr, map_size);
185		return NULL;
186	}
187
188	return map_ptr;
189}
190
191static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout)
192{
193	struct timespec cur_time;
194	unsigned long cur_scan, init_scan;
195
196	if (ksm_read_sysfs(KSM_FP("full_scans"), &init_scan))
197		return 1;
198	cur_scan = init_scan;
199
200	while (cur_scan < init_scan + scan_count) {
201		if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan))
202			return 1;
203		if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) {
204			perror("clock_gettime");
205			return 1;
206		}
207		if ((cur_time.tv_sec - start_time.tv_sec) > timeout) {
208			printf("Scan time limit exceeded\n");
209			return 1;
210		}
211	}
212
213	return 0;
214}
215
216static int ksm_merge_pages(int merge_type, void *addr, size_t size,
217			struct timespec start_time, int timeout)
218{
219	if (merge_type == KSM_MERGE_MADVISE) {
220		if (madvise(addr, size, MADV_MERGEABLE)) {
221			perror("madvise");
222			return 1;
223		}
224	} else if (merge_type == KSM_MERGE_PRCTL) {
225		if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) {
226			perror("prctl");
227			return 1;
228		}
229	}
230
231	if (ksm_write_sysfs(KSM_FP("run"), 1))
232		return 1;
233
234	/* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */
235	if (ksm_do_scan(2, start_time, timeout))
236		return 1;
237
238	return 0;
239}
240
241static int ksm_unmerge_pages(void *addr, size_t size,
242			     struct timespec start_time, int timeout)
243{
244	if (madvise(addr, size, MADV_UNMERGEABLE)) {
245		perror("madvise");
246		return 1;
247	}
248	return 0;
249}
250
251static bool assert_ksm_pages_count(long dupl_page_count)
252{
253	unsigned long max_page_sharing, pages_sharing, pages_shared;
254
255	if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||
256	    ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||
257	    ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing))
258		return false;
259
260	if (debug) {
261		ksm_print_sysfs();
262		ksm_print_procfs();
263	}
264
265	/*
266	 * Since there must be at least 2 pages for merging and 1 page can be
267	 * shared with the limited number of pages (max_page_sharing), sometimes
268	 * there are 'leftover' pages that cannot be merged. For example, if there
269	 * are 11 pages and max_page_sharing = 10, then only 10 pages will be
270	 * merged and the 11th page won't be affected. As a result, when the number
271	 * of duplicate pages is divided by max_page_sharing and the remainder is 1,
272	 * pages_shared and pages_sharing values will be equal between dupl_page_count
273	 * and dupl_page_count - 1.
274	 */
275	if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) {
276		if (pages_shared == dupl_page_count / max_page_sharing &&
277		    pages_sharing == pages_shared * (max_page_sharing - 1))
278			return true;
279	} else {
280		if (pages_shared == (dupl_page_count / max_page_sharing + 1) &&
281		    pages_sharing == dupl_page_count - pages_shared)
282			return true;
283	}
284
285	return false;
286}
287
288static int ksm_save_def(struct ksm_sysfs *ksm_sysfs)
289{
290	if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) ||
291	    numa_available() ? 0 :
292		ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) ||
293	    ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) ||
294	    ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) ||
295	    ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) ||
296	    ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
297			   &ksm_sysfs->stable_node_chains_prune_millisecs) ||
298	    ksm_read_sysfs(KSM_FP("use_zero_pages"), &ksm_sysfs->use_zero_pages))
299		return 1;
300
301	return 0;
302}
303
304static int ksm_restore(struct ksm_sysfs *ksm_sysfs)
305{
306	if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) ||
307	    numa_available() ? 0 :
308		ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) ||
309	    ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) ||
310	    ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) ||
311	    ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) ||
312	    ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
313			    ksm_sysfs->stable_node_chains_prune_millisecs) ||
314	    ksm_write_sysfs(KSM_FP("use_zero_pages"), ksm_sysfs->use_zero_pages))
315		return 1;
316
317	return 0;
318}
319
320static int check_ksm_merge(int merge_type, int mapping, int prot,
321			long page_count, int timeout, size_t page_size)
322{
323	void *map_ptr;
324	struct timespec start_time;
325
326	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
327		perror("clock_gettime");
328		return KSFT_FAIL;
329	}
330
331	/* fill pages with the same data and merge them */
332	map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
333	if (!map_ptr)
334		return KSFT_FAIL;
335
336	if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
337		goto err_out;
338
339	/* verify that the right number of pages are merged */
340	if (assert_ksm_pages_count(page_count)) {
341		printf("OK\n");
342		munmap(map_ptr, page_size * page_count);
343		if (merge_type == KSM_MERGE_PRCTL)
344			prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0);
345		return KSFT_PASS;
346	}
347
348err_out:
349	printf("Not OK\n");
350	munmap(map_ptr, page_size * page_count);
351	return KSFT_FAIL;
352}
353
354static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, size_t page_size)
355{
356	void *map_ptr;
357	struct timespec start_time;
358	int page_count = 2;
359
360	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
361		perror("clock_gettime");
362		return KSFT_FAIL;
363	}
364
365	/* fill pages with the same data and merge them */
366	map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
367	if (!map_ptr)
368		return KSFT_FAIL;
369
370	if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
371		goto err_out;
372
373	/* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */
374	memset(map_ptr, '-', 1);
375	memset(map_ptr + page_size, '+', 1);
376
377	/* get at least 1 scan, so KSM can detect that the pages were modified */
378	if (ksm_do_scan(1, start_time, timeout))
379		goto err_out;
380
381	/* check that unmerging was successful and 0 pages are currently merged */
382	if (assert_ksm_pages_count(0)) {
383		printf("OK\n");
384		munmap(map_ptr, page_size * page_count);
385		return KSFT_PASS;
386	}
387
388err_out:
389	printf("Not OK\n");
390	munmap(map_ptr, page_size * page_count);
391	return KSFT_FAIL;
392}
393
394static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long page_count,
395				int timeout, bool use_zero_pages, size_t page_size)
396{
397	void *map_ptr;
398	struct timespec start_time;
399
400	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
401		perror("clock_gettime");
402		return KSFT_FAIL;
403	}
404
405	if (ksm_write_sysfs(KSM_FP("use_zero_pages"), use_zero_pages))
406		return KSFT_FAIL;
407
408	/* fill pages with zero and try to merge them */
409	map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count);
410	if (!map_ptr)
411		return KSFT_FAIL;
412
413	if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
414		goto err_out;
415
416       /*
417	* verify that the right number of pages are merged:
418	* 1) if use_zero_pages is set to 1, empty pages are merged
419	*    with the kernel zero page instead of with each other;
420	* 2) if use_zero_pages is set to 0, empty pages are not treated specially
421	*    and merged as usual.
422	*/
423	if (use_zero_pages && !assert_ksm_pages_count(0))
424		goto err_out;
425	else if (!use_zero_pages && !assert_ksm_pages_count(page_count))
426		goto err_out;
427
428	printf("OK\n");
429	munmap(map_ptr, page_size * page_count);
430	return KSFT_PASS;
431
432err_out:
433	printf("Not OK\n");
434	munmap(map_ptr, page_size * page_count);
435	return KSFT_FAIL;
436}
437
438static int get_next_mem_node(int node)
439{
440
441	long node_size;
442	int mem_node = 0;
443	int i, max_node = numa_max_node();
444
445	for (i = node + 1; i <= max_node + node; i++) {
446		mem_node = i % (max_node + 1);
447		node_size = numa_node_size(mem_node, NULL);
448		if (node_size > 0)
449			break;
450	}
451	return mem_node;
452}
453
454static int get_first_mem_node(void)
455{
456	return get_next_mem_node(numa_max_node());
457}
458
459static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeout,
460				bool merge_across_nodes, size_t page_size)
461{
462	void *numa1_map_ptr, *numa2_map_ptr;
463	struct timespec start_time;
464	int page_count = 2;
465	int first_node;
466
467	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
468		perror("clock_gettime");
469		return KSFT_FAIL;
470	}
471
472	if (numa_available() < 0) {
473		perror("NUMA support not enabled");
474		return KSFT_SKIP;
475	}
476	if (numa_num_configured_nodes() <= 1) {
477		printf("At least 2 NUMA nodes must be available\n");
478		return KSFT_SKIP;
479	}
480	if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes))
481		return KSFT_FAIL;
482
483	/* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */
484	first_node = get_first_mem_node();
485	numa1_map_ptr = numa_alloc_onnode(page_size, first_node);
486	numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node));
487	if (!numa1_map_ptr || !numa2_map_ptr) {
488		perror("numa_alloc_onnode");
489		return KSFT_FAIL;
490	}
491
492	memset(numa1_map_ptr, '*', page_size);
493	memset(numa2_map_ptr, '*', page_size);
494
495	/* try to merge the pages */
496	if (ksm_merge_pages(merge_type, numa1_map_ptr, page_size, start_time, timeout) ||
497	    ksm_merge_pages(merge_type, numa2_map_ptr, page_size, start_time, timeout))
498		goto err_out;
499
500       /*
501	* verify that the right number of pages are merged:
502	* 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged;
503	* 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is
504	*    only 1 unique page in each node and they can't be shared.
505	*/
506	if (merge_across_nodes && !assert_ksm_pages_count(page_count))
507		goto err_out;
508	else if (!merge_across_nodes && !assert_ksm_pages_count(0))
509		goto err_out;
510
511	numa_free(numa1_map_ptr, page_size);
512	numa_free(numa2_map_ptr, page_size);
513	printf("OK\n");
514	return KSFT_PASS;
515
516err_out:
517	numa_free(numa1_map_ptr, page_size);
518	numa_free(numa2_map_ptr, page_size);
519	printf("Not OK\n");
520	return KSFT_FAIL;
521}
522
523static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot,
524				int timeout, size_t map_size)
525{
526	void *map_ptr, *map_ptr_orig;
527	struct timespec start_time, end_time;
528	unsigned long scan_time_ns;
529	int pagemap_fd, n_normal_pages, n_huge_pages;
530
531	if (!thp_is_enabled()) {
532		printf("Transparent Hugepages not available\n");
533		return KSFT_SKIP;
534	}
535
536	map_size *= MB;
537	size_t len = map_size;
538
539	len -= len % HPAGE_SIZE;
540	map_ptr_orig = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE,
541			MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0);
542	map_ptr = map_ptr_orig + HPAGE_SIZE - (uintptr_t)map_ptr_orig % HPAGE_SIZE;
543
544	if (map_ptr_orig == MAP_FAILED)
545		err(2, "initial mmap");
546
547	if (madvise(map_ptr, len, MADV_HUGEPAGE))
548		err(2, "MADV_HUGEPAGE");
549
550	pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
551	if (pagemap_fd < 0)
552		err(2, "open pagemap");
553
554	n_normal_pages = 0;
555	n_huge_pages = 0;
556	for (void *p = map_ptr; p < map_ptr + len; p += HPAGE_SIZE) {
557		if (allocate_transhuge(p, pagemap_fd) < 0)
558			n_normal_pages++;
559		else
560			n_huge_pages++;
561	}
562	printf("Number of normal pages:    %d\n", n_normal_pages);
563	printf("Number of huge pages:    %d\n", n_huge_pages);
564
565	memset(map_ptr, '*', len);
566
567	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
568		perror("clock_gettime");
569		goto err_out;
570	}
571	if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
572		goto err_out;
573	if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
574		perror("clock_gettime");
575		goto err_out;
576	}
577
578	scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
579		       (end_time.tv_nsec - start_time.tv_nsec);
580
581	printf("Total size:    %lu MiB\n", map_size / MB);
582	printf("Total time:    %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
583	       scan_time_ns % NSEC_PER_SEC);
584	printf("Average speed:  %.3f MiB/s\n", (map_size / MB) /
585					       ((double)scan_time_ns / NSEC_PER_SEC));
586
587	munmap(map_ptr_orig, len + HPAGE_SIZE);
588	return KSFT_PASS;
589
590err_out:
591	printf("Not OK\n");
592	munmap(map_ptr_orig, len + HPAGE_SIZE);
593	return KSFT_FAIL;
594}
595
596static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size)
597{
598	void *map_ptr;
599	struct timespec start_time, end_time;
600	unsigned long scan_time_ns;
601
602	map_size *= MB;
603
604	map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);
605	if (!map_ptr)
606		return KSFT_FAIL;
607
608	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
609		perror("clock_gettime");
610		goto err_out;
611	}
612	if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
613		goto err_out;
614	if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
615		perror("clock_gettime");
616		goto err_out;
617	}
618
619	scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
620		       (end_time.tv_nsec - start_time.tv_nsec);
621
622	printf("Total size:    %lu MiB\n", map_size / MB);
623	printf("Total time:    %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
624	       scan_time_ns % NSEC_PER_SEC);
625	printf("Average speed:  %.3f MiB/s\n", (map_size / MB) /
626					       ((double)scan_time_ns / NSEC_PER_SEC));
627
628	munmap(map_ptr, map_size);
629	return KSFT_PASS;
630
631err_out:
632	printf("Not OK\n");
633	munmap(map_ptr, map_size);
634	return KSFT_FAIL;
635}
636
637static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size)
638{
639	void *map_ptr;
640	struct timespec start_time, end_time;
641	unsigned long scan_time_ns;
642
643	map_size *= MB;
644
645	map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);
646	if (!map_ptr)
647		return KSFT_FAIL;
648	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
649		perror("clock_gettime");
650		goto err_out;
651	}
652	if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
653		goto err_out;
654
655	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
656		perror("clock_gettime");
657		goto err_out;
658	}
659	if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout))
660		goto err_out;
661	if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
662		perror("clock_gettime");
663		goto err_out;
664	}
665
666	scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
667		       (end_time.tv_nsec - start_time.tv_nsec);
668
669	printf("Total size:    %lu MiB\n", map_size / MB);
670	printf("Total time:    %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
671	       scan_time_ns % NSEC_PER_SEC);
672	printf("Average speed:  %.3f MiB/s\n", (map_size / MB) /
673					       ((double)scan_time_ns / NSEC_PER_SEC));
674
675	munmap(map_ptr, map_size);
676	return KSFT_PASS;
677
678err_out:
679	printf("Not OK\n");
680	munmap(map_ptr, map_size);
681	return KSFT_FAIL;
682}
683
684static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size_t page_size)
685{
686	void *map_ptr;
687	struct timespec start_time, end_time;
688	unsigned long cow_time_ns;
689
690	/* page_count must be less than 2*page_size */
691	size_t page_count = 4000;
692
693	map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
694	if (!map_ptr)
695		return KSFT_FAIL;
696
697	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
698		perror("clock_gettime");
699		return KSFT_FAIL;
700	}
701	for (size_t i = 0; i < page_count - 1; i = i + 2)
702		memset(map_ptr + page_size * i, '-', 1);
703	if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
704		perror("clock_gettime");
705		return KSFT_FAIL;
706	}
707
708	cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
709		       (end_time.tv_nsec - start_time.tv_nsec);
710
711	printf("Total size:    %lu MiB\n\n", (page_size * page_count) / MB);
712	printf("Not merged pages:\n");
713	printf("Total time:     %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,
714	       cow_time_ns % NSEC_PER_SEC);
715	printf("Average speed:  %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) /
716					       ((double)cow_time_ns / NSEC_PER_SEC));
717
718	/* Create 2000 pairs of duplicate pages */
719	for (size_t i = 0; i < page_count - 1; i = i + 2) {
720		memset(map_ptr + page_size * i, '+', i / 2 + 1);
721		memset(map_ptr + page_size * (i + 1), '+', i / 2 + 1);
722	}
723	if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
724		goto err_out;
725
726	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
727		perror("clock_gettime");
728		goto err_out;
729	}
730	for (size_t i = 0; i < page_count - 1; i = i + 2)
731		memset(map_ptr + page_size * i, '-', 1);
732	if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
733		perror("clock_gettime");
734		goto err_out;
735	}
736
737	cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
738		       (end_time.tv_nsec - start_time.tv_nsec);
739
740	printf("Merged pages:\n");
741	printf("Total time:     %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,
742	       cow_time_ns % NSEC_PER_SEC);
743	printf("Average speed:  %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) /
744					       ((double)cow_time_ns / NSEC_PER_SEC));
745
746	munmap(map_ptr, page_size * page_count);
747	return KSFT_PASS;
748
749err_out:
750	printf("Not OK\n");
751	munmap(map_ptr, page_size * page_count);
752	return KSFT_FAIL;
753}
754
755int main(int argc, char *argv[])
756{
757	int ret = 0, opt;
758	int prot = 0;
759	int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT;
760	int merge_type = KSM_MERGE_TYPE_DEFAULT;
761	long page_count = KSM_PAGE_COUNT_DEFAULT;
762	size_t page_size = sysconf(_SC_PAGESIZE);
763	struct ksm_sysfs ksm_sysfs_old;
764	int test_name = CHECK_KSM_MERGE;
765	bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT;
766	bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT;
767	long size_MB = 0;
768
769	while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD")) != -1) {
770		switch (opt) {
771		case 'a':
772			prot = str_to_prot(optarg);
773			break;
774		case 'p':
775			page_count = atol(optarg);
776			if (page_count <= 0) {
777				printf("The number of pages must be greater than 0\n");
778				return KSFT_FAIL;
779			}
780			break;
781		case 'l':
782			ksm_scan_limit_sec = atoi(optarg);
783			if (ksm_scan_limit_sec <= 0) {
784				printf("Timeout value must be greater than 0\n");
785				return KSFT_FAIL;
786			}
787			break;
788		case 'h':
789			print_help();
790			break;
791		case 'z':
792			if (strcmp(optarg, "0") == 0)
793				use_zero_pages = 0;
794			else
795				use_zero_pages = 1;
796			break;
797		case 'm':
798			if (strcmp(optarg, "0") == 0)
799				merge_across_nodes = 0;
800			else
801				merge_across_nodes = 1;
802			break;
803		case 'd':
804			debug = 1;
805			break;
806		case 's':
807			size_MB = atoi(optarg);
808			if (size_MB <= 0) {
809				printf("Size must be greater than 0\n");
810				return KSFT_FAIL;
811			}
812			break;
813		case 't':
814			{
815				int tmp = atoi(optarg);
816
817				if (tmp < 0 || tmp > KSM_MERGE_LAST) {
818					printf("Invalid merge type\n");
819					return KSFT_FAIL;
820				}
821				merge_type = tmp;
822			}
823			break;
824		case 'M':
825			break;
826		case 'U':
827			test_name = CHECK_KSM_UNMERGE;
828			break;
829		case 'Z':
830			test_name = CHECK_KSM_ZERO_PAGE_MERGE;
831			break;
832		case 'N':
833			test_name = CHECK_KSM_NUMA_MERGE;
834			break;
835		case 'P':
836			test_name = KSM_MERGE_TIME;
837			break;
838		case 'H':
839			test_name = KSM_MERGE_TIME_HUGE_PAGES;
840			break;
841		case 'D':
842			test_name = KSM_UNMERGE_TIME;
843			break;
844		case 'C':
845			test_name = KSM_COW_TIME;
846			break;
847		default:
848			return KSFT_FAIL;
849		}
850	}
851
852	if (prot == 0)
853		prot = str_to_prot(KSM_PROT_STR_DEFAULT);
854
855	if (access(KSM_SYSFS_PATH, F_OK)) {
856		printf("Config KSM not enabled\n");
857		return KSFT_SKIP;
858	}
859
860	if (ksm_save_def(&ksm_sysfs_old)) {
861		printf("Cannot save default tunables\n");
862		return KSFT_FAIL;
863	}
864
865	if (ksm_write_sysfs(KSM_FP("run"), 2) ||
866	    ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) ||
867	    numa_available() ? 0 :
868		ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) ||
869	    ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count))
870		return KSFT_FAIL;
871
872	switch (test_name) {
873	case CHECK_KSM_MERGE:
874		ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count,
875				      ksm_scan_limit_sec, page_size);
876		break;
877	case CHECK_KSM_UNMERGE:
878		ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
879					ksm_scan_limit_sec, page_size);
880		break;
881	case CHECK_KSM_ZERO_PAGE_MERGE:
882		ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
883						page_count, ksm_scan_limit_sec, use_zero_pages,
884						page_size);
885		break;
886	case CHECK_KSM_NUMA_MERGE:
887		ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
888					ksm_scan_limit_sec, merge_across_nodes, page_size);
889		break;
890	case KSM_MERGE_TIME:
891		if (size_MB == 0) {
892			printf("Option '-s' is required.\n");
893			return KSFT_FAIL;
894		}
895		ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
896				ksm_scan_limit_sec, size_MB);
897		break;
898	case KSM_MERGE_TIME_HUGE_PAGES:
899		if (size_MB == 0) {
900			printf("Option '-s' is required.\n");
901			return KSFT_FAIL;
902		}
903		ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
904				ksm_scan_limit_sec, size_MB);
905		break;
906	case KSM_UNMERGE_TIME:
907		if (size_MB == 0) {
908			printf("Option '-s' is required.\n");
909			return KSFT_FAIL;
910		}
911		ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
912				       ksm_scan_limit_sec, size_MB);
913		break;
914	case KSM_COW_TIME:
915		ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
916				ksm_scan_limit_sec, page_size);
917		break;
918	}
919
920	if (ksm_restore(&ksm_sysfs_old)) {
921		printf("Cannot restore default tunables\n");
922		return KSFT_FAIL;
923	}
924
925	return ret;
926}