x86_energy_perf_policy.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * x86_energy_perf_policy -- set the energy versus performance
   4 * policy preference bias on recent X86 processors.
   5 */
   6/*
   7 * Copyright (c) 2010 - 2025 Intel Corporation.
   8 * Len Brown <len.brown@intel.com>
   9 */
  10
  11#define _GNU_SOURCE
  12#include MSRHEADER
  13#include <stdio.h>
  14#include <unistd.h>
  15#include <sys/types.h>
  16#include <sched.h>
  17#include <sys/stat.h>
  18#include <sys/resource.h>
  19#include <getopt.h>
  20#include <err.h>
  21#include <fcntl.h>
  22#include <signal.h>
  23#include <sys/time.h>
  24#include <limits.h>
  25#include <stdlib.h>
  26#include <string.h>
  27#include <cpuid.h>
  28#include <errno.h>
  29
  30#define	OPTARG_NORMAL			(INT_MAX - 1)
  31#define	OPTARG_POWER			(INT_MAX - 2)
  32#define	OPTARG_BALANCE_POWER		(INT_MAX - 3)
  33#define	OPTARG_BALANCE_PERFORMANCE	(INT_MAX - 4)
  34#define	OPTARG_PERFORMANCE		(INT_MAX - 5)
  35
  36struct msr_hwp_cap {
  37	unsigned char highest;
  38	unsigned char guaranteed;
  39	unsigned char efficient;
  40	unsigned char lowest;
  41};
  42
  43struct msr_hwp_request {
  44	unsigned char hwp_min;
  45	unsigned char hwp_max;
  46	unsigned char hwp_desired;
  47	unsigned char hwp_epp;
  48	unsigned int hwp_window;
  49	unsigned char hwp_use_pkg;
  50} req_update;
  51
  52unsigned int debug;
  53unsigned int verbose;
  54unsigned int force;
  55char *progname;
  56int base_cpu;
  57unsigned char update_epb;
  58unsigned long long new_epb;
  59unsigned char turbo_is_enabled;
  60unsigned char update_turbo;
  61unsigned char turbo_update_value;
  62unsigned char update_hwp_epp;
  63unsigned char update_hwp_min;
  64unsigned char update_hwp_max;
  65unsigned char hwp_limits_done_via_sysfs;
  66unsigned char update_hwp_desired;
  67unsigned char update_hwp_window;
  68unsigned char update_hwp_use_pkg;
  69unsigned char update_hwp_enable;
  70#define hwp_update_enabled() (update_hwp_enable | update_hwp_epp | update_hwp_max | update_hwp_min | update_hwp_desired | update_hwp_window | update_hwp_use_pkg)
  71int max_cpu_num;
  72int max_pkg_num;
  73#define MAX_PACKAGES 64
  74unsigned int first_cpu_in_pkg[MAX_PACKAGES];
  75unsigned long long pkg_present_set;
  76unsigned long long pkg_selected_set;
  77cpu_set_t *cpu_present_set;
  78cpu_set_t *cpu_selected_set;
  79int genuine_intel;
  80
  81size_t cpu_setsize;
  82
  83char *proc_stat = "/proc/stat";
  84
  85unsigned int has_epb;	/* MSR_IA32_ENERGY_PERF_BIAS */
  86unsigned int has_hwp;	/* IA32_PM_ENABLE, IA32_HWP_CAPABILITIES */
  87			/* IA32_HWP_REQUEST, IA32_HWP_STATUS */
  88unsigned int has_hwp_notify;		/* IA32_HWP_INTERRUPT */
  89unsigned int has_hwp_activity_window;	/* IA32_HWP_REQUEST[bits 41:32] */
  90unsigned int has_hwp_epp;	/* IA32_HWP_REQUEST[bits 31:24] */
  91unsigned int has_hwp_request_pkg;	/* IA32_HWP_REQUEST_PKG */
  92
  93unsigned int bdx_highest_ratio;
  94
  95#define PATH_TO_CPU "/sys/devices/system/cpu/"
  96#define SYSFS_PATH_MAX 255
  97
  98static int use_android_msr_path;
  99
 100/*
 101 * maintain compatibility with original implementation, but don't document it:
 102 */
 103void usage(void)
 104{
 105	fprintf(stderr, "%s [options] [scope][field value]\n", progname);
 106	fprintf(stderr, "scope: --cpu cpu-list [--hwp-use-pkg #] | --pkg pkg-list\n");
 107	fprintf(stderr, "field: --all | --epb | --hwp-epp | --hwp-min | --hwp-max | --hwp-desired\n");
 108	fprintf(stderr, "other: --hwp-enable | --turbo-enable (0 | 1) | --help | --force\n");
 109	fprintf(stderr,
 110		"value: ( # | \"normal\" | \"performance\" | \"balance-performance\" | \"balance-power\"| \"power\")\n");
 111	fprintf(stderr, "--hwp-window usec\n");
 112
 113	fprintf(stderr, "Specify only Energy Performance BIAS (legacy usage):\n");
 114	fprintf(stderr, "%s: [-c cpu] [-v] (-r | policy-value )\n", progname);
 115
 116	exit(1);
 117}
 118
 119/*
 120 * If bdx_highest_ratio is set,
 121 * then we must translate between MSR format and simple ratio
 122 * used on the cmdline.
 123 */
 124int ratio_2_msr_perf(int ratio)
 125{
 126	int msr_perf;
 127
 128	if (!bdx_highest_ratio)
 129		return ratio;
 130
 131	msr_perf = ratio * 255 / bdx_highest_ratio;
 132
 133	if (debug)
 134		fprintf(stderr, "%d = ratio_to_msr_perf(%d)\n", msr_perf, ratio);
 135
 136	return msr_perf;
 137}
 138int msr_perf_2_ratio(int msr_perf)
 139{
 140	int ratio;
 141	double d;
 142
 143	if (!bdx_highest_ratio)
 144		return msr_perf;
 145
 146	d = (double)msr_perf * (double) bdx_highest_ratio / 255.0;
 147	d = d + 0.5;	/* round */
 148	ratio = (int)d;
 149
 150	if (debug)
 151		fprintf(stderr, "%d = msr_perf_ratio(%d) {%f}\n", ratio, msr_perf, d);
 152
 153	return ratio;
 154}
 155int parse_cmdline_epb(int i)
 156{
 157	if (!has_epb)
 158		errx(1, "EPB not enabled on this platform");
 159
 160	update_epb = 1;
 161
 162	switch (i) {
 163	case OPTARG_POWER:
 164		return ENERGY_PERF_BIAS_POWERSAVE;
 165	case OPTARG_BALANCE_POWER:
 166		return ENERGY_PERF_BIAS_BALANCE_POWERSAVE;
 167	case OPTARG_NORMAL:
 168		return ENERGY_PERF_BIAS_NORMAL;
 169	case OPTARG_BALANCE_PERFORMANCE:
 170		return ENERGY_PERF_BIAS_BALANCE_PERFORMANCE;
 171	case OPTARG_PERFORMANCE:
 172		return ENERGY_PERF_BIAS_PERFORMANCE;
 173	}
 174	if (i < 0 || i > ENERGY_PERF_BIAS_POWERSAVE)
 175		errx(1, "--epb must be from 0 to 15");
 176	return i;
 177}
 178
 179#define HWP_CAP_LOWEST 0
 180#define HWP_CAP_HIGHEST 255
 181
 182/*
 183 * "performance" changes hwp_min to cap.highest
 184 * All others leave it at cap.lowest
 185 */
 186int parse_cmdline_hwp_min(int i)
 187{
 188	update_hwp_min = 1;
 189
 190	switch (i) {
 191	case OPTARG_POWER:
 192	case OPTARG_BALANCE_POWER:
 193	case OPTARG_NORMAL:
 194	case OPTARG_BALANCE_PERFORMANCE:
 195		return HWP_CAP_LOWEST;
 196	case OPTARG_PERFORMANCE:
 197		return HWP_CAP_HIGHEST;
 198	}
 199	return i;
 200}
 201/*
 202 * "power" changes hwp_max to cap.lowest
 203 * All others leave it at cap.highest
 204 */
 205int parse_cmdline_hwp_max(int i)
 206{
 207	update_hwp_max = 1;
 208
 209	switch (i) {
 210	case OPTARG_POWER:
 211		return HWP_CAP_LOWEST;
 212	case OPTARG_NORMAL:
 213	case OPTARG_BALANCE_POWER:
 214	case OPTARG_BALANCE_PERFORMANCE:
 215	case OPTARG_PERFORMANCE:
 216		return HWP_CAP_HIGHEST;
 217	}
 218	return i;
 219}
 220/*
 221 * for --hwp-des, all strings leave it in autonomous mode
 222 * If you want to change it, you need to explicitly pick a value
 223 */
 224int parse_cmdline_hwp_desired(int i)
 225{
 226	update_hwp_desired = 1;
 227
 228	switch (i) {
 229	case OPTARG_POWER:
 230	case OPTARG_BALANCE_POWER:
 231	case OPTARG_BALANCE_PERFORMANCE:
 232	case OPTARG_NORMAL:
 233	case OPTARG_PERFORMANCE:
 234		return 0;	/* autonomous */
 235	}
 236	return i;
 237}
 238
 239int parse_cmdline_hwp_window(int i)
 240{
 241	unsigned int exponent;
 242
 243	update_hwp_window = 1;
 244
 245	switch (i) {
 246	case OPTARG_POWER:
 247	case OPTARG_BALANCE_POWER:
 248	case OPTARG_NORMAL:
 249	case OPTARG_BALANCE_PERFORMANCE:
 250	case OPTARG_PERFORMANCE:
 251		return 0;
 252	}
 253	if (i < 0 || i > 1270000000) {
 254		fprintf(stderr, "--hwp-window: 0 for auto; 1 - 1270000000 usec for window duration\n");
 255		usage();
 256	}
 257	for (exponent = 0; ; ++exponent) {
 258		if (debug)
 259			printf("%d 10^%d\n", i, exponent);
 260
 261		if (i <= 127)
 262			break;
 263
 264		i = i / 10;
 265	}
 266	if (debug)
 267		fprintf(stderr, "%d*10^%d: 0x%x\n", i, exponent, (exponent << 7) | i);
 268
 269	return (exponent << 7) | i;
 270}
 271int parse_cmdline_hwp_epp(int i)
 272{
 273	update_hwp_epp = 1;
 274
 275	switch (i) {
 276	case OPTARG_POWER:
 277		return HWP_EPP_POWERSAVE;
 278	case OPTARG_BALANCE_POWER:
 279		return HWP_EPP_BALANCE_POWERSAVE;
 280	case OPTARG_NORMAL:
 281	case OPTARG_BALANCE_PERFORMANCE:
 282		return HWP_EPP_BALANCE_PERFORMANCE;
 283	case OPTARG_PERFORMANCE:
 284		return HWP_EPP_PERFORMANCE;
 285	}
 286	if (i < 0 || i > 0xff) {
 287		fprintf(stderr, "--hwp-epp must be from 0 to 0xff\n");
 288		usage();
 289	}
 290	return i;
 291}
 292int parse_cmdline_turbo(int i)
 293{
 294	update_turbo = 1;
 295
 296	switch (i) {
 297	case OPTARG_POWER:
 298		return 0;
 299	case OPTARG_NORMAL:
 300	case OPTARG_BALANCE_POWER:
 301	case OPTARG_BALANCE_PERFORMANCE:
 302	case OPTARG_PERFORMANCE:
 303		return 1;
 304	}
 305	if (i < 0 || i > 1) {
 306		fprintf(stderr, "--turbo-enable: 1 to enable, 0 to disable\n");
 307		usage();
 308	}
 309	return i;
 310}
 311
 312int parse_optarg_string(char *s)
 313{
 314	int i;
 315	char *endptr;
 316
 317	if (!strncmp(s, "default", 7))
 318		return OPTARG_NORMAL;
 319
 320	if (!strncmp(s, "normal", 6))
 321		return OPTARG_NORMAL;
 322
 323	if (!strncmp(s, "power", 9))
 324		return OPTARG_POWER;
 325
 326	if (!strncmp(s, "balance-power", 17))
 327		return OPTARG_BALANCE_POWER;
 328
 329	if (!strncmp(s, "balance-performance", 19))
 330		return OPTARG_BALANCE_PERFORMANCE;
 331
 332	if (!strncmp(s, "performance", 11))
 333		return OPTARG_PERFORMANCE;
 334
 335	i = strtol(s, &endptr, 0);
 336	if (s == endptr) {
 337		fprintf(stderr, "no digits in \"%s\"\n", s);
 338		usage();
 339	}
 340	if (i == LONG_MIN || i == LONG_MAX)
 341		errx(-1, "%s", s);
 342
 343	if (i > 0xFF)
 344		errx(-1, "%d (0x%x) must be < 256", i, i);
 345
 346	if (i < 0)
 347		errx(-1, "%d (0x%x) must be >= 0", i, i);
 348	return i;
 349}
 350
 351void parse_cmdline_all(char *s)
 352{
 353	force++;
 354	update_hwp_enable = 1;
 355	req_update.hwp_min = parse_cmdline_hwp_min(parse_optarg_string(s));
 356	req_update.hwp_max = parse_cmdline_hwp_max(parse_optarg_string(s));
 357	req_update.hwp_epp = parse_cmdline_hwp_epp(parse_optarg_string(s));
 358	if (has_epb)
 359		new_epb = parse_cmdline_epb(parse_optarg_string(s));
 360	turbo_update_value = parse_cmdline_turbo(parse_optarg_string(s));
 361	req_update.hwp_desired = parse_cmdline_hwp_desired(parse_optarg_string(s));
 362	req_update.hwp_window = parse_cmdline_hwp_window(parse_optarg_string(s));
 363}
 364
 365void validate_cpu_selected_set(void)
 366{
 367	int cpu;
 368
 369	if (CPU_COUNT_S(cpu_setsize, cpu_selected_set) == 0)
 370		errx(0, "no CPUs requested");
 371
 372	for (cpu = 0; cpu <= max_cpu_num; ++cpu) {
 373		if (CPU_ISSET_S(cpu, cpu_setsize, cpu_selected_set))
 374			if (!CPU_ISSET_S(cpu, cpu_setsize, cpu_present_set))
 375				errx(1, "Requested cpu%d is not present", cpu);
 376	}
 377}
 378
 379void parse_cmdline_cpu(char *s)
 380{
 381	char *startp, *endp;
 382	int cpu = 0;
 383
 384	if (pkg_selected_set) {
 385		usage();
 386		errx(1, "--cpu | --pkg");
 387	}
 388	cpu_selected_set = CPU_ALLOC((max_cpu_num + 1));
 389	if (cpu_selected_set == NULL)
 390		err(1, "cpu_selected_set");
 391	CPU_ZERO_S(cpu_setsize, cpu_selected_set);
 392
 393	for (startp = s; startp && *startp;) {
 394
 395		if (*startp == ',') {
 396			startp++;
 397			continue;
 398		}
 399
 400		if (*startp == '-') {
 401			int end_cpu;
 402
 403			startp++;
 404			end_cpu = strtol(startp, &endp, 10);
 405			if (startp == endp)
 406				continue;
 407
 408			while (cpu <= end_cpu) {
 409				if (cpu > max_cpu_num)
 410					errx(1, "Requested cpu%d exceeds max cpu%d", cpu, max_cpu_num);
 411				CPU_SET_S(cpu, cpu_setsize, cpu_selected_set);
 412				cpu++;
 413			}
 414			startp = endp;
 415			continue;
 416		}
 417
 418		if (strncmp(startp, "all", 3) == 0) {
 419			for (cpu = 0; cpu <= max_cpu_num; cpu += 1) {
 420				if (CPU_ISSET_S(cpu, cpu_setsize, cpu_present_set))
 421					CPU_SET_S(cpu, cpu_setsize, cpu_selected_set);
 422			}
 423			startp += 3;
 424			if (*startp == 0)
 425				break;
 426		}
 427		/* "--cpu even" is not documented */
 428		if (strncmp(startp, "even", 4) == 0) {
 429			for (cpu = 0; cpu <= max_cpu_num; cpu += 2) {
 430				if (CPU_ISSET_S(cpu, cpu_setsize, cpu_present_set))
 431					CPU_SET_S(cpu, cpu_setsize, cpu_selected_set);
 432			}
 433			startp += 4;
 434			if (*startp == 0)
 435				break;
 436		}
 437
 438		/* "--cpu odd" is not documented */
 439		if (strncmp(startp, "odd", 3) == 0) {
 440			for (cpu = 1; cpu <= max_cpu_num; cpu += 2) {
 441				if (CPU_ISSET_S(cpu, cpu_setsize, cpu_present_set))
 442					CPU_SET_S(cpu, cpu_setsize, cpu_selected_set);
 443			}
 444			startp += 3;
 445			if (*startp == 0)
 446				break;
 447		}
 448
 449		cpu = strtol(startp, &endp, 10);
 450		if (startp == endp)
 451			errx(1, "--cpu cpu-set: confused by '%s'", startp);
 452		if (cpu > max_cpu_num)
 453			errx(1, "Requested cpu%d exceeds max cpu%d", cpu, max_cpu_num);
 454		CPU_SET_S(cpu, cpu_setsize, cpu_selected_set);
 455		startp = endp;
 456	}
 457
 458	validate_cpu_selected_set();
 459
 460}
 461
 462void parse_cmdline_pkg(char *s)
 463{
 464	char *startp, *endp;
 465	int pkg = 0;
 466
 467	if (cpu_selected_set) {
 468		usage();
 469		errx(1, "--pkg | --cpu");
 470	}
 471	pkg_selected_set = 0;
 472
 473	for (startp = s; startp && *startp;) {
 474
 475		if (*startp == ',') {
 476			startp++;
 477			continue;
 478		}
 479
 480		if (*startp == '-') {
 481			int end_pkg;
 482
 483			startp++;
 484			end_pkg = strtol(startp, &endp, 10);
 485			if (startp == endp)
 486				continue;
 487
 488			while (pkg <= end_pkg) {
 489				if (pkg > max_pkg_num)
 490					errx(1, "Requested pkg%d exceeds max pkg%d", pkg, max_pkg_num);
 491				pkg_selected_set |= 1 << pkg;
 492				pkg++;
 493			}
 494			startp = endp;
 495			continue;
 496		}
 497
 498		if (strncmp(startp, "all", 3) == 0) {
 499			pkg_selected_set = pkg_present_set;
 500			return;
 501		}
 502
 503		pkg = strtol(startp, &endp, 10);
 504		if (pkg > max_pkg_num)
 505			errx(1, "Requested pkg%d Exceeds max pkg%d", pkg, max_pkg_num);
 506		pkg_selected_set |= 1 << pkg;
 507		startp = endp;
 508	}
 509}
 510
 511void for_packages(unsigned long long pkg_set, int (func)(int))
 512{
 513	int pkg_num;
 514
 515	for (pkg_num = 0; pkg_num <= max_pkg_num; ++pkg_num) {
 516		if (pkg_set & (1UL << pkg_num))
 517			func(pkg_num);
 518	}
 519}
 520
 521void print_version(void)
 522{
 523	printf("x86_energy_perf_policy 2025.11.22 Len Brown <lenb@kernel.org>\n");
 524}
 525
 526void cmdline(int argc, char **argv)
 527{
 528	int opt;
 529	int option_index = 0;
 530
 531	static struct option long_options[] = {
 532		{"all",		required_argument,	0, 'a'},
 533		{"cpu",		required_argument,	0, 'c'},
 534		{"pkg",		required_argument,	0, 'p'},
 535		{"debug",	no_argument,		0, 'd'},
 536		{"hwp-desired",	required_argument,	0, 'D'},
 537		{"epb",	required_argument,	0, 'B'},
 538		{"force",	no_argument,	0, 'f'},
 539		{"hwp-enable",	no_argument,	0, 'e'},
 540		{"help",	no_argument,	0, 'h'},
 541		{"hwp-epp",	required_argument,	0, 'P'},
 542		{"hwp-min",	required_argument,	0, 'm'},
 543		{"hwp-max",	required_argument,	0, 'M'},
 544		{"read",	no_argument,		0, 'r'},
 545		{"turbo-enable",	required_argument,	0, 't'},
 546		{"hwp-use-pkg",	required_argument,	0, 'u'},
 547		{"version",	no_argument,		0, 'v'},
 548		{"hwp-window",	required_argument,	0, 'w'},
 549		{0,		0,			0, 0 }
 550	};
 551
 552	progname = argv[0];
 553
 554	while ((opt = getopt_long_only(argc, argv, "+a:c:dD:E:e:f:m:M:rt:u:vw:",
 555				long_options, &option_index)) != -1) {
 556		switch (opt) {
 557		case 'a':
 558			parse_cmdline_all(optarg);
 559			break;
 560		case 'B':
 561			new_epb = parse_cmdline_epb(parse_optarg_string(optarg));
 562			break;
 563		case 'c':
 564			parse_cmdline_cpu(optarg);
 565			break;
 566		case 'e':
 567			update_hwp_enable = 1;
 568			break;
 569		case 'h':
 570			usage();
 571			break;
 572		case 'd':
 573			debug++;
 574			verbose++;
 575			break;
 576		case 'f':
 577			force++;
 578			break;
 579		case 'D':
 580			req_update.hwp_desired = parse_cmdline_hwp_desired(parse_optarg_string(optarg));
 581			break;
 582		case 'm':
 583			req_update.hwp_min = parse_cmdline_hwp_min(parse_optarg_string(optarg));
 584			break;
 585		case 'M':
 586			req_update.hwp_max = parse_cmdline_hwp_max(parse_optarg_string(optarg));
 587			break;
 588		case 'p':
 589			parse_cmdline_pkg(optarg);
 590			break;
 591		case 'P':
 592			req_update.hwp_epp = parse_cmdline_hwp_epp(parse_optarg_string(optarg));
 593			break;
 594		case 'r':
 595			/* v1 used -r to specify read-only mode, now the default */
 596			break;
 597		case 't':
 598			turbo_update_value = parse_cmdline_turbo(parse_optarg_string(optarg));
 599			break;
 600		case 'u':
 601			update_hwp_use_pkg++;
 602			if (atoi(optarg) == 0)
 603				req_update.hwp_use_pkg = 0;
 604			else
 605				req_update.hwp_use_pkg = 1;
 606			break;
 607		case 'v':
 608			print_version();
 609			exit(0);
 610			break;
 611		case 'w':
 612			req_update.hwp_window = parse_cmdline_hwp_window(parse_optarg_string(optarg));
 613			break;
 614		default:
 615			usage();
 616		}
 617	}
 618	/*
 619	 * v1 allowed "performance"|"normal"|"power" with no policy specifier
 620	 * to update BIAS.  Continue to support that, even though no longer documented.
 621	 */
 622	if (argc == optind + 1)
 623		new_epb = parse_cmdline_epb(parse_optarg_string(argv[optind]));
 624
 625	if (argc > optind + 1) {
 626		fprintf(stderr, "stray parameter '%s'\n", argv[optind + 1]);
 627		usage();
 628	}
 629}
 630
 631/*
 632 * Open a file, and exit on failure
 633 */
 634FILE *fopen_or_die(const char *path, const char *mode)
 635{
 636	FILE *filep = fopen(path, mode);
 637
 638	if (!filep)
 639		err(1, "%s: open failed", path);
 640	return filep;
 641}
 642
 643void err_on_hypervisor(void)
 644{
 645	FILE *cpuinfo;
 646	char *flags, *hypervisor;
 647	char *buffer;
 648
 649	/* On VMs /proc/cpuinfo contains a "flags" entry for hypervisor */
 650	cpuinfo = fopen_or_die("/proc/cpuinfo", "r");
 651
 652	buffer = malloc(4096);
 653	if (!buffer) {
 654		fclose(cpuinfo);
 655		err(-ENOMEM, "buffer malloc fail");
 656	}
 657
 658	if (!fread(buffer, 1024, 1, cpuinfo)) {
 659		fclose(cpuinfo);
 660		free(buffer);
 661		err(1, "Reading /proc/cpuinfo failed");
 662	}
 663
 664	flags = strstr(buffer, "flags");
 665	if (!flags) {
 666		fclose(cpuinfo);
 667		free(buffer);
 668		err(1, "Failed to find 'flags' in /proc/cpuinfo");
 669	}
 670	rewind(cpuinfo);
 671	fseek(cpuinfo, flags - buffer, SEEK_SET);
 672	if (!fgets(buffer, 4096, cpuinfo)) {
 673		fclose(cpuinfo);
 674		free(buffer);
 675		err(1, "Reading /proc/cpuinfo failed");
 676	}
 677	fclose(cpuinfo);
 678
 679	hypervisor = strstr(buffer, "hypervisor");
 680
 681	free(buffer);
 682
 683	if (hypervisor)
 684		err(-1,
 685		    "not supported on this virtual machine");
 686}
 687
 688int get_msr(int cpu, int offset, unsigned long long *msr)
 689{
 690	int retval;
 691	char pathname[32];
 692	int fd;
 693
 694	sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", cpu);
 695	fd = open(pathname, O_RDONLY);
 696	if (fd < 0)
 697		err(-1, "%s open failed, try chown or chmod +r %s, or run as root",
 698		   pathname, use_android_msr_path ? "/dev/msr*" : "/dev/cpu/*/msr");
 699
 700
 701	retval = pread(fd, msr, sizeof(*msr), offset);
 702	if (retval != sizeof(*msr)) {
 703		err_on_hypervisor();
 704		err(-1, "%s offset 0x%llx read failed", pathname, (unsigned long long)offset);
 705	}
 706
 707	if (debug > 1)
 708		fprintf(stderr, "get_msr(cpu%d, 0x%X, 0x%llX)\n", cpu, offset, *msr);
 709
 710	close(fd);
 711	return 0;
 712}
 713
 714int put_msr(int cpu, int offset, unsigned long long new_msr)
 715{
 716	char pathname[32];
 717	int retval;
 718	int fd;
 719
 720	sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", cpu);
 721	fd = open(pathname, O_RDWR);
 722	if (fd < 0)
 723		err(-1, "%s open failed, try chown or chmod +r %s, or run as root",
 724		   pathname, use_android_msr_path ? "/dev/msr*" : "/dev/cpu/*/msr");
 725
 726	retval = pwrite(fd, &new_msr, sizeof(new_msr), offset);
 727	if (retval != sizeof(new_msr))
 728		err(-2, "pwrite(cpu%d, offset 0x%x, 0x%llx) = %d", cpu, offset, new_msr, retval);
 729
 730	close(fd);
 731
 732	if (debug > 1)
 733		fprintf(stderr, "put_msr(cpu%d, 0x%X, 0x%llX)\n", cpu, offset, new_msr);
 734
 735	return 0;
 736}
 737
 738static unsigned int read_sysfs(const char *path, char *buf, size_t buflen)
 739{
 740	ssize_t numread;
 741	int fd;
 742
 743	fd = open(path, O_RDONLY);
 744	if (fd == -1)
 745		return 0;
 746
 747	numread = read(fd, buf, buflen - 1);
 748	if (numread < 1) {
 749		close(fd);
 750		return 0;
 751	}
 752
 753	buf[numread] = '\0';
 754	close(fd);
 755
 756	return (unsigned int) numread;
 757}
 758
 759static unsigned int write_sysfs(const char *path, char *buf, size_t buflen)
 760{
 761	ssize_t numwritten;
 762	int fd;
 763
 764	fd = open(path, O_WRONLY);
 765	if (fd == -1)
 766		return 0;
 767
 768	numwritten = write(fd, buf, buflen - 1);
 769	if (numwritten < 1) {
 770		perror("write failed\n");
 771		close(fd);
 772		return -1;
 773	}
 774
 775	close(fd);
 776
 777	return (unsigned int) numwritten;
 778}
 779
 780void print_hwp_cap(int cpu, struct msr_hwp_cap *cap, char *str)
 781{
 782	if (cpu != -1)
 783		printf("cpu%d: ", cpu);
 784
 785	printf("HWP_CAP: low %d eff %d guar %d high %d\n",
 786		cap->lowest, cap->efficient, cap->guaranteed, cap->highest);
 787}
 788void read_hwp_cap(int cpu, struct msr_hwp_cap *cap, unsigned int msr_offset)
 789{
 790	unsigned long long msr;
 791
 792	get_msr(cpu, msr_offset, &msr);
 793
 794	cap->highest = msr_perf_2_ratio(HWP_HIGHEST_PERF(msr));
 795	cap->guaranteed = msr_perf_2_ratio(HWP_GUARANTEED_PERF(msr));
 796	cap->efficient = msr_perf_2_ratio(HWP_MOSTEFFICIENT_PERF(msr));
 797	cap->lowest = msr_perf_2_ratio(HWP_LOWEST_PERF(msr));
 798}
 799
 800void print_hwp_request(int cpu, struct msr_hwp_request *h, char *str)
 801{
 802	if (cpu != -1)
 803		printf("cpu%d: ", cpu);
 804
 805	if (str)
 806		printf("%s", str);
 807
 808	printf("HWP_REQ: min %d max %d des %d epp %d window 0x%x (%d*10^%dus) use_pkg %d\n",
 809		h->hwp_min, h->hwp_max, h->hwp_desired, h->hwp_epp,
 810		h->hwp_window, h->hwp_window & 0x7F, (h->hwp_window >> 7) & 0x7, h->hwp_use_pkg);
 811}
 812void print_hwp_request_pkg(int pkg, struct msr_hwp_request *h, char *str)
 813{
 814	printf("pkg%d: ", pkg);
 815
 816	if (str)
 817		printf("%s", str);
 818
 819	printf("HWP_REQ_PKG: min %d max %d des %d epp %d window 0x%x (%d*10^%dus)\n",
 820		h->hwp_min, h->hwp_max, h->hwp_desired, h->hwp_epp,
 821		h->hwp_window, h->hwp_window & 0x7F, (h->hwp_window >> 7) & 0x7);
 822}
 823void read_hwp_request_msr(int cpu, struct msr_hwp_request *hwp_req, unsigned int msr_offset)
 824{
 825	unsigned long long msr;
 826
 827	get_msr(cpu, msr_offset, &msr);
 828
 829	hwp_req->hwp_min = msr_perf_2_ratio((((msr) >> 0) & 0xff));
 830	hwp_req->hwp_max = msr_perf_2_ratio((((msr) >> 8) & 0xff));
 831	hwp_req->hwp_desired = msr_perf_2_ratio((((msr) >> 16) & 0xff));
 832	hwp_req->hwp_epp = (((msr) >> 24) & 0xff);
 833	hwp_req->hwp_window = (((msr) >> 32) & 0x3ff);
 834	hwp_req->hwp_use_pkg = (((msr) >> 42) & 0x1);
 835}
 836
 837void write_hwp_request_msr(int cpu, struct msr_hwp_request *hwp_req, unsigned int msr_offset)
 838{
 839	unsigned long long msr = 0;
 840
 841	if (debug > 1)
 842		printf("cpu%d: requesting min %d max %d des %d epp %d window 0x%0x use_pkg %d\n",
 843			cpu, hwp_req->hwp_min, hwp_req->hwp_max,
 844			hwp_req->hwp_desired, hwp_req->hwp_epp,
 845			hwp_req->hwp_window, hwp_req->hwp_use_pkg);
 846
 847	msr |= HWP_MIN_PERF(ratio_2_msr_perf(hwp_req->hwp_min));
 848	msr |= HWP_MAX_PERF(ratio_2_msr_perf(hwp_req->hwp_max));
 849	msr |= HWP_DESIRED_PERF(ratio_2_msr_perf(hwp_req->hwp_desired));
 850	msr |= HWP_ENERGY_PERF_PREFERENCE(hwp_req->hwp_epp);
 851	msr |= HWP_ACTIVITY_WINDOW(hwp_req->hwp_window);
 852	msr |= HWP_PACKAGE_CONTROL(hwp_req->hwp_use_pkg);
 853
 854	put_msr(cpu, msr_offset, msr);
 855}
 856
 857static int get_epb_sysfs(int cpu)
 858{
 859	char path[SYSFS_PATH_MAX];
 860	char linebuf[3];
 861	char *endp;
 862	long val;
 863
 864	if (!has_epb)
 865		return -1;
 866
 867	snprintf(path, sizeof(path), PATH_TO_CPU "cpu%u/power/energy_perf_bias", cpu);
 868
 869	if (!read_sysfs(path, linebuf, 3))
 870		return -1;
 871
 872	val = strtol(linebuf, &endp, 0);
 873	if (endp == linebuf || errno == ERANGE)
 874		return -1;
 875
 876	return (int)val;
 877}
 878
 879static int set_epb_sysfs(int cpu, int val)
 880{
 881	char path[SYSFS_PATH_MAX];
 882	char linebuf[3];
 883	char *endp;
 884	int ret;
 885
 886	if (!has_epb)
 887		return -1;
 888
 889	snprintf(path, sizeof(path), PATH_TO_CPU "cpu%u/power/energy_perf_bias", cpu);
 890	snprintf(linebuf, sizeof(linebuf), "%d", val);
 891
 892	ret = write_sysfs(path, linebuf, 3);
 893	if (ret <= 0)
 894		return -1;
 895
 896	val = strtol(linebuf, &endp, 0);
 897	if (endp == linebuf || errno == ERANGE)
 898		return -1;
 899
 900	return (int)val;
 901}
 902
 903int print_cpu_msrs(int cpu)
 904{
 905	struct msr_hwp_request req;
 906	struct msr_hwp_cap cap;
 907	int epb;
 908
 909	epb = get_epb_sysfs(cpu);
 910	if (epb >= 0)
 911		printf("cpu%d: EPB %u\n", cpu, (unsigned int) epb);
 912
 913	if (!has_hwp)
 914		return 0;
 915
 916	read_hwp_request_msr(cpu, &req, MSR_HWP_REQUEST);
 917	print_hwp_request(cpu, &req, "");
 918
 919	read_hwp_cap(cpu, &cap, MSR_HWP_CAPABILITIES);
 920	print_hwp_cap(cpu, &cap, "");
 921
 922	return 0;
 923}
 924
 925int print_pkg_msrs(int pkg)
 926{
 927	struct msr_hwp_request req;
 928	unsigned long long msr;
 929
 930	if (!has_hwp)
 931		return 0;
 932
 933	read_hwp_request_msr(first_cpu_in_pkg[pkg], &req, MSR_HWP_REQUEST_PKG);
 934	print_hwp_request_pkg(pkg, &req, "");
 935
 936	if (has_hwp_notify) {
 937		get_msr(first_cpu_in_pkg[pkg], MSR_HWP_INTERRUPT, &msr);
 938		fprintf(stderr,
 939		"pkg%d: MSR_HWP_INTERRUPT: 0x%08llx (Excursion_Min-%sabled, Guaranteed_Perf_Change-%sabled)\n",
 940		pkg, msr,
 941		((msr) & 0x2) ? "EN" : "Dis",
 942		((msr) & 0x1) ? "EN" : "Dis");
 943	}
 944	get_msr(first_cpu_in_pkg[pkg], MSR_HWP_STATUS, &msr);
 945	fprintf(stderr,
 946		"pkg%d: MSR_HWP_STATUS: 0x%08llx (%sExcursion_Min, %sGuaranteed_Perf_Change)\n",
 947		pkg, msr,
 948		((msr) & 0x4) ? "" : "No-",
 949		((msr) & 0x1) ? "" : "No-");
 950
 951	return 0;
 952}
 953
 954/*
 955 * Assumption: All HWP systems have 100 MHz bus clock
 956 */
 957int ratio_2_sysfs_khz(int ratio)
 958{
 959	int bclk_khz = 100 * 1000;	/* 100,000 KHz = 100 MHz */
 960
 961	return ratio * bclk_khz;
 962}
 963/*
 964 * If HWP is enabled and cpufreq sysfs attribtes are present,
 965 * then update via sysfs. The intel_pstate driver may modify (clip)
 966 * this request, say, when HWP_CAP is outside of PLATFORM_INFO limits,
 967 * and the driver-chosen value takes precidence.
 968 *
 969 * (intel_pstate's max_perf_pct and min_perf_pct will follow cpufreq,
 970 *  so we don't have to touch that.)
 971 */
 972void update_cpufreq_scaling_freq(int is_max, int cpu, unsigned int ratio)
 973{
 974	char pathname[64];
 975	FILE *fp;
 976	int retval;
 977	int khz;
 978
 979	sprintf(pathname, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_%s_freq",
 980		cpu, is_max ? "max" : "min");
 981
 982	fp = fopen(pathname, "w");
 983	if (!fp) {
 984		if (debug)
 985			perror(pathname);
 986		return;
 987	}
 988
 989	khz = ratio_2_sysfs_khz(ratio);
 990	retval = fprintf(fp, "%d", khz);
 991	if (retval < 0)
 992		if (debug)
 993			perror("fprintf");
 994	if (debug)
 995		printf("echo %d > %s\n", khz, pathname);
 996
 997	fclose(fp);
 998}
 999
1000/*
1001 * We update all sysfs before updating any MSRs because of
1002 * bugs in cpufreq/intel_pstate where the sysfs writes
1003 * for a CPU may change the min/max values on other CPUS.
1004 */
1005
1006int update_sysfs(int cpu)
1007{
1008	if (!has_hwp)
1009		return 0;
1010
1011	if (!hwp_update_enabled())
1012		return 0;
1013
1014	if (access("/sys/devices/system/cpu/cpu0/cpufreq", F_OK))
1015		return 0;
1016
1017	if (update_hwp_min)
1018		update_cpufreq_scaling_freq(0, cpu, req_update.hwp_min);
1019
1020	if (update_hwp_max)
1021		update_cpufreq_scaling_freq(1, cpu, req_update.hwp_max);
1022
1023	hwp_limits_done_via_sysfs = 1;
1024
1025	return 0;
1026}
1027
1028int verify_hwp_req_self_consistency(int cpu, struct msr_hwp_request *req)
1029{
1030	/* fail if min > max requested */
1031	if (req->hwp_min > req->hwp_max) {
1032		errx(1, "cpu%d: requested hwp-min %d > hwp_max %d",
1033			cpu, req->hwp_min, req->hwp_max);
1034	}
1035
1036	/* fail if desired > max requestd */
1037	if (req->hwp_desired && (req->hwp_desired > req->hwp_max)) {
1038		errx(1, "cpu%d: requested hwp-desired %d > hwp_max %d",
1039			cpu, req->hwp_desired, req->hwp_max);
1040	}
1041	/* fail if desired < min requestd */
1042	if (req->hwp_desired && (req->hwp_desired < req->hwp_min)) {
1043		errx(1, "cpu%d: requested hwp-desired %d < requested hwp_min %d",
1044			cpu, req->hwp_desired, req->hwp_min);
1045	}
1046
1047	return 0;
1048}
1049
1050int check_hwp_request_v_hwp_capabilities(int cpu, struct msr_hwp_request *req, struct msr_hwp_cap *cap)
1051{
1052	if (update_hwp_max) {
1053		if (req->hwp_max > cap->highest)
1054			errx(1, "cpu%d: requested max %d > capabilities highest %d, use --force?",
1055				cpu, req->hwp_max, cap->highest);
1056		if (req->hwp_max < cap->lowest)
1057			errx(1, "cpu%d: requested max %d < capabilities lowest %d, use --force?",
1058				cpu, req->hwp_max, cap->lowest);
1059	}
1060
1061	if (update_hwp_min) {
1062		if (req->hwp_min > cap->highest)
1063			errx(1, "cpu%d: requested min %d > capabilities highest %d, use --force?",
1064				cpu, req->hwp_min, cap->highest);
1065		if (req->hwp_min < cap->lowest)
1066			errx(1, "cpu%d: requested min %d < capabilities lowest %d, use --force?",
1067				cpu, req->hwp_min, cap->lowest);
1068	}
1069
1070	if (update_hwp_min && update_hwp_max && (req->hwp_min > req->hwp_max))
1071		errx(1, "cpu%d: requested min %d > requested max %d",
1072			cpu, req->hwp_min, req->hwp_max);
1073
1074	if (update_hwp_desired && req->hwp_desired) {
1075		if (req->hwp_desired > req->hwp_max)
1076			errx(1, "cpu%d: requested desired %d > requested max %d, use --force?",
1077				cpu, req->hwp_desired, req->hwp_max);
1078		if (req->hwp_desired < req->hwp_min)
1079			errx(1, "cpu%d: requested desired %d < requested min %d, use --force?",
1080				cpu, req->hwp_desired, req->hwp_min);
1081		if (req->hwp_desired < cap->lowest)
1082			errx(1, "cpu%d: requested desired %d < capabilities lowest %d, use --force?",
1083				cpu, req->hwp_desired, cap->lowest);
1084		if (req->hwp_desired > cap->highest)
1085			errx(1, "cpu%d: requested desired %d > capabilities highest %d, use --force?",
1086				cpu, req->hwp_desired, cap->highest);
1087	}
1088
1089	return 0;
1090}
1091
1092int update_hwp_request_msr(int cpu)
1093{
1094	struct msr_hwp_request req;
1095	struct msr_hwp_cap cap;
1096
1097	int msr_offset = MSR_HWP_REQUEST;
1098
1099	read_hwp_request_msr(cpu, &req, msr_offset);
1100	if (debug)
1101		print_hwp_request(cpu, &req, "old: ");
1102
1103	if (update_hwp_min && !hwp_limits_done_via_sysfs)
1104		req.hwp_min = req_update.hwp_min;
1105
1106	if (update_hwp_max && !hwp_limits_done_via_sysfs)
1107		req.hwp_max = req_update.hwp_max;
1108
1109	if (update_hwp_desired)
1110		req.hwp_desired = req_update.hwp_desired;
1111
1112	if (update_hwp_window)
1113		req.hwp_window = req_update.hwp_window;
1114
1115	if (update_hwp_epp)
1116		req.hwp_epp = req_update.hwp_epp;
1117
1118	req.hwp_use_pkg = req_update.hwp_use_pkg;
1119
1120	read_hwp_cap(cpu, &cap, MSR_HWP_CAPABILITIES);
1121	if (debug)
1122		print_hwp_cap(cpu, &cap, "");
1123
1124	if (!force)
1125		check_hwp_request_v_hwp_capabilities(cpu, &req, &cap);
1126
1127	verify_hwp_req_self_consistency(cpu, &req);
1128
1129	write_hwp_request_msr(cpu, &req, msr_offset);
1130
1131	if (debug) {
1132		read_hwp_request_msr(cpu, &req, msr_offset);
1133		print_hwp_request(cpu, &req, "new: ");
1134	}
1135	return 0;
1136}
1137int update_hwp_request_pkg_msr(int pkg)
1138{
1139	struct msr_hwp_request req;
1140	struct msr_hwp_cap cap;
1141	int cpu = first_cpu_in_pkg[pkg];
1142
1143	int msr_offset = MSR_HWP_REQUEST_PKG;
1144
1145	read_hwp_request_msr(cpu, &req, msr_offset);
1146	if (debug)
1147		print_hwp_request_pkg(pkg, &req, "old: ");
1148
1149	if (update_hwp_min)
1150		req.hwp_min = req_update.hwp_min;
1151
1152	if (update_hwp_max)
1153		req.hwp_max = req_update.hwp_max;
1154
1155	if (update_hwp_desired)
1156		req.hwp_desired = req_update.hwp_desired;
1157
1158	if (update_hwp_window)
1159		req.hwp_window = req_update.hwp_window;
1160
1161	if (update_hwp_epp)
1162		req.hwp_epp = req_update.hwp_epp;
1163
1164	read_hwp_cap(cpu, &cap, MSR_HWP_CAPABILITIES);
1165	if (debug)
1166		print_hwp_cap(cpu, &cap, "");
1167
1168	if (!force)
1169		check_hwp_request_v_hwp_capabilities(cpu, &req, &cap);
1170
1171	verify_hwp_req_self_consistency(cpu, &req);
1172
1173	write_hwp_request_msr(cpu, &req, msr_offset);
1174
1175	if (debug) {
1176		read_hwp_request_msr(cpu, &req, msr_offset);
1177		print_hwp_request_pkg(pkg, &req, "new: ");
1178	}
1179	return 0;
1180}
1181
1182int enable_hwp_on_cpu(int cpu)
1183{
1184	unsigned long long old_msr, new_msr;
1185
1186	get_msr(cpu, MSR_PM_ENABLE, &old_msr);
1187
1188	if (old_msr & 1)
1189		return 0;	/* already enabled */
1190
1191	new_msr = old_msr | 1;
1192	put_msr(cpu, MSR_PM_ENABLE, new_msr);
1193
1194	if (verbose)
1195		printf("cpu%d: MSR_PM_ENABLE old: %llX new: %llX\n", cpu, old_msr, new_msr);
1196
1197	return 0;
1198}
1199
1200int update_cpu_epb_sysfs(int cpu)
1201{
1202	int epb;
1203
1204	epb = get_epb_sysfs(cpu);
1205	set_epb_sysfs(cpu, new_epb);
1206
1207	if (verbose)
1208		printf("cpu%d: ENERGY_PERF_BIAS old: %d new: %d\n",
1209			cpu, epb, (unsigned int) new_epb);
1210
1211	return 0;
1212}
1213
1214int update_cpu_msrs(int cpu)
1215{
1216	unsigned long long msr;
1217
1218	if (update_turbo) {
1219		int turbo_is_present_and_disabled;
1220
1221		get_msr(cpu, MSR_IA32_MISC_ENABLE, &msr);
1222
1223		turbo_is_present_and_disabled = ((msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE) != 0);
1224
1225		if (turbo_update_value == 1)	{
1226			if (turbo_is_present_and_disabled) {
1227				msr &= ~MSR_IA32_MISC_ENABLE_TURBO_DISABLE;
1228				put_msr(cpu, MSR_IA32_MISC_ENABLE, msr);
1229				if (verbose)
1230					printf("cpu%d: turbo ENABLE\n", cpu);
1231			}
1232		} else {
1233			/*
1234			 * if "turbo_is_enabled" were known to be describe this cpu
1235			 * then we could use it here to skip redundant disable requests.
1236			 * but cpu may be in a different package, so we always write.
1237			 */
1238			msr |= MSR_IA32_MISC_ENABLE_TURBO_DISABLE;
1239			put_msr(cpu, MSR_IA32_MISC_ENABLE, msr);
1240			if (verbose)
1241				printf("cpu%d: turbo DISABLE\n", cpu);
1242		}
1243	}
1244
1245	if (!has_hwp)
1246		return 0;
1247
1248	if (!hwp_update_enabled())
1249		return 0;
1250
1251	update_hwp_request_msr(cpu);
1252	return 0;
1253}
1254
1255unsigned int get_pkg_num(int cpu)
1256{
1257	FILE *fp;
1258	char pathname[128];
1259	unsigned int pkg;
1260	int retval;
1261
1262	sprintf(pathname, "/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu);
1263
1264	fp = fopen_or_die(pathname, "r");
1265	retval = fscanf(fp, "%d\n", &pkg);
1266	if (retval != 1)
1267		errx(1, "%s: failed to parse", pathname);
1268	fclose(fp);
1269	return pkg;
1270}
1271
1272int set_max_cpu_pkg_num(int cpu)
1273{
1274	unsigned int pkg;
1275
1276	if (max_cpu_num < cpu)
1277		max_cpu_num = cpu;
1278
1279	pkg = get_pkg_num(cpu);
1280
1281	if (pkg >= MAX_PACKAGES)
1282		errx(1, "cpu%d: %d >= MAX_PACKAGES (%d)", cpu, pkg, MAX_PACKAGES);
1283
1284	if (pkg > max_pkg_num)
1285		max_pkg_num = pkg;
1286
1287	if ((pkg_present_set & (1ULL << pkg)) == 0) {
1288		pkg_present_set |= (1ULL << pkg);
1289		first_cpu_in_pkg[pkg] = cpu;
1290	}
1291
1292	return 0;
1293}
1294int mark_cpu_present(int cpu)
1295{
1296	CPU_SET_S(cpu, cpu_setsize, cpu_present_set);
1297	return 0;
1298}
1299
1300/*
1301 * run func(cpu) on every cpu in /proc/stat
1302 * return max_cpu number
1303 */
1304int for_all_proc_cpus(int (func)(int))
1305{
1306	FILE *fp;
1307	int cpu_num;
1308	int retval;
1309
1310	fp = fopen_or_die(proc_stat, "r");
1311
1312	retval = fscanf(fp, "cpu %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n");
1313	if (retval != 0)
1314		err(1, "%s: failed to parse format", proc_stat);
1315
1316	while (1) {
1317		retval = fscanf(fp, "cpu%u %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n", &cpu_num);
1318		if (retval != 1)
1319			break;
1320
1321		retval = func(cpu_num);
1322		if (retval) {
1323			fclose(fp);
1324			return retval;
1325		}
1326	}
1327	fclose(fp);
1328	return 0;
1329}
1330
1331void for_all_cpus_in_set(size_t set_size, cpu_set_t *cpu_set, int (func)(int))
1332{
1333	int cpu_num;
1334
1335	for (cpu_num = 0; cpu_num <= max_cpu_num; ++cpu_num)
1336		if (CPU_ISSET_S(cpu_num, set_size, cpu_set))
1337			func(cpu_num);
1338}
1339int for_all_cpus_in_set_and(size_t set_size, cpu_set_t *cpu_set, int (func)(int))
1340{
1341	int cpu_num;
1342	int retval = 1;
1343
1344	for (cpu_num = 0; cpu_num <= max_cpu_num; ++cpu_num)
1345		if (CPU_ISSET_S(cpu_num, set_size, cpu_set))
1346			retval &= func(cpu_num);
1347
1348	return retval;
1349}
1350
1351void init_data_structures(void)
1352{
1353	for_all_proc_cpus(set_max_cpu_pkg_num);
1354
1355	cpu_setsize = CPU_ALLOC_SIZE((max_cpu_num + 1));
1356
1357	cpu_present_set = CPU_ALLOC((max_cpu_num + 1));
1358	if (cpu_present_set == NULL)
1359		err(3, "CPU_ALLOC");
1360	CPU_ZERO_S(cpu_setsize, cpu_present_set);
1361	for_all_proc_cpus(mark_cpu_present);
1362}
1363
1364int is_hwp_enabled_on_cpu(int cpu_num)
1365{
1366	unsigned long long msr;
1367	int retval;
1368
1369	/* MSR_PM_ENABLE[1] == 1 if HWP is enabled and MSRs visible */
1370	get_msr(cpu_num, MSR_PM_ENABLE, &msr);
1371	retval = (msr & 1);
1372
1373	if (verbose)
1374		fprintf(stderr, "cpu%d: %sHWP\n", cpu_num, retval ? "" : "No-");
1375
1376	return retval;
1377}
1378
1379/*
1380 * verify_hwp_is_enabled()
1381 *
1382 * Set (has_hwp=0) if no HWP feature or any of selected CPU set does not have HWP enabled
1383 */
1384void verify_hwp_is_enabled(void)
1385{
1386	int retval;
1387
1388	if (!has_hwp)	/* set in early_cpuid() */
1389		return;
1390
1391	retval = for_all_cpus_in_set_and(cpu_setsize, cpu_selected_set, is_hwp_enabled_on_cpu);
1392
1393	if (retval == 0) {
1394		fprintf(stderr, "HWP can be enabled using '--hwp-enable'\n");
1395		has_hwp = 0;
1396	}
1397}
1398
1399int req_update_bounds_check(void)
1400{
1401	if (!hwp_update_enabled())
1402		return 0;
1403
1404	/* fail if min > max requested */
1405	if ((update_hwp_max && update_hwp_min) &&
1406	    (req_update.hwp_min > req_update.hwp_max)) {
1407		printf("hwp-min %d > hwp_max %d\n", req_update.hwp_min, req_update.hwp_max);
1408		return -EINVAL;
1409	}
1410
1411	/* fail if desired > max requestd */
1412	if (req_update.hwp_desired && update_hwp_max &&
1413	    (req_update.hwp_desired > req_update.hwp_max)) {
1414		printf("hwp-desired cannot be greater than hwp_max\n");
1415		return -EINVAL;
1416	}
1417	/* fail if desired < min requestd */
1418	if (req_update.hwp_desired && update_hwp_min &&
1419	    (req_update.hwp_desired < req_update.hwp_min)) {
1420		printf("hwp-desired cannot be less than hwp_min\n");
1421		return -EINVAL;
1422	}
1423
1424	return 0;
1425}
1426
1427void set_base_cpu(void)
1428{
1429	base_cpu = sched_getcpu();
1430	if (base_cpu < 0)
1431		err(-ENODEV, "No valid cpus found");
1432}
1433
1434static void probe_android_msr_path(void)
1435{
1436	struct stat sb;
1437	char test_path[32];
1438
1439	sprintf(test_path, "/dev/msr%d", base_cpu);
1440	if (stat(test_path, &sb) == 0)
1441		use_android_msr_path = 1;
1442}
1443
1444void probe_dev_msr(void)
1445{
1446	struct stat sb;
1447	char pathname[32];
1448
1449	probe_android_msr_path();
1450
1451	sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", base_cpu);
1452	if (stat(pathname, &sb)) {
1453		if (system("/sbin/modprobe msr > /dev/null 2>&1")) {
1454			if (use_android_msr_path)
1455				err(-5, "no /dev/msr0, Try \"# modprobe msr\" ");
1456			else
1457				err(-5, "no /dev/cpu/0/msr, Try \"# modprobe msr\" ");
1458		}
1459	}
1460}
1461
1462static void get_cpuid_or_exit(unsigned int leaf,
1463			     unsigned int *eax, unsigned int *ebx,
1464			     unsigned int *ecx, unsigned int *edx)
1465{
1466	if (!__get_cpuid(leaf, eax, ebx, ecx, edx))
1467		errx(1, "Processor not supported\n");
1468}
1469
1470/*
1471 * early_cpuid()
1472 * initialize turbo_is_enabled, has_hwp, has_epb
1473 * before cmdline is parsed
1474 */
1475void early_cpuid(void)
1476{
1477	unsigned int eax, ebx, ecx, edx;
1478	unsigned int fms, family, model;
1479
1480	get_cpuid_or_exit(1, &fms, &ebx, &ecx, &edx);
1481	family = (fms >> 8) & 0xf;
1482	model = (fms >> 4) & 0xf;
1483	if (family == 6 || family == 0xf)
1484		model += ((fms >> 16) & 0xf) << 4;
1485
1486	if (model == 0x4F) {
1487		unsigned long long msr;
1488
1489		get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT, &msr);
1490
1491		bdx_highest_ratio = msr & 0xFF;
1492	}
1493
1494	get_cpuid_or_exit(0x6, &eax, &ebx, &ecx, &edx);
1495	turbo_is_enabled = (eax >> 1) & 1;
1496	has_hwp = (eax >> 7) & 1;
1497	has_epb = (ecx >> 3) & 1;
1498}
1499
1500/*
1501 * parse_cpuid()
1502 * set
1503 * has_hwp, has_hwp_notify, has_hwp_activity_window, has_hwp_epp, has_hwp_request_pkg, has_epb
1504 */
1505void parse_cpuid(void)
1506{
1507	unsigned int eax, ebx, ecx, edx, max_level;
1508	unsigned int fms, family, model, stepping;
1509
1510	eax = ebx = ecx = edx = 0;
1511
1512	get_cpuid_or_exit(0, &max_level, &ebx, &ecx, &edx);
1513
1514	if (ebx == 0x756e6547 && edx == 0x49656e69 && ecx == 0x6c65746e)
1515		genuine_intel = 1;
1516
1517	if (debug)
1518		fprintf(stderr, "CPUID(0): %.4s%.4s%.4s ",
1519			(char *)&ebx, (char *)&edx, (char *)&ecx);
1520
1521	get_cpuid_or_exit(1, &fms, &ebx, &ecx, &edx);
1522	family = (fms >> 8) & 0xf;
1523	model = (fms >> 4) & 0xf;
1524	stepping = fms & 0xf;
1525	if (family == 6 || family == 0xf)
1526		model += ((fms >> 16) & 0xf) << 4;
1527
1528	if (debug) {
1529		fprintf(stderr, "%d CPUID levels; family:model:stepping 0x%x:%x:%x (%d:%d:%d)\n",
1530			max_level, family, model, stepping, family, model, stepping);
1531		fprintf(stderr, "CPUID(1): %s %s %s %s %s %s %s %s\n",
1532			ecx & (1 << 0) ? "SSE3" : "-",
1533			ecx & (1 << 3) ? "MONITOR" : "-",
1534			ecx & (1 << 7) ? "EIST" : "-",
1535			ecx & (1 << 8) ? "TM2" : "-",
1536			edx & (1 << 4) ? "TSC" : "-",
1537			edx & (1 << 5) ? "MSR" : "-",
1538			edx & (1 << 22) ? "ACPI-TM" : "-",
1539			edx & (1 << 29) ? "TM" : "-");
1540	}
1541
1542	if (!(edx & (1 << 5)))
1543		errx(1, "CPUID: no MSR");
1544
1545
1546	get_cpuid_or_exit(0x6, &eax, &ebx, &ecx, &edx);
1547	/* turbo_is_enabled already set */
1548	/* has_hwp already set */
1549	has_hwp_notify = eax & (1 << 8);
1550	has_hwp_activity_window = eax & (1 << 9);
1551	has_hwp_epp = eax & (1 << 10);
1552	has_hwp_request_pkg = eax & (1 << 11);
1553
1554	if (!has_hwp_request_pkg && update_hwp_use_pkg)
1555		errx(1, "--hwp-use-pkg is not available on this hardware");
1556
1557	/* has_epb already set */
1558
1559	if (debug)
1560		fprintf(stderr,
1561			"CPUID(6): %sTURBO, %sHWP, %sHWPnotify, %sHWPwindow, %sHWPepp, %sHWPpkg, %sEPB\n",
1562			turbo_is_enabled ? "" : "No-",
1563			has_hwp ? "" : "No-",
1564			has_hwp_notify ? "" : "No-",
1565			has_hwp_activity_window ? "" : "No-",
1566			has_hwp_epp ? "" : "No-",
1567			has_hwp_request_pkg ? "" : "No-",
1568			has_epb ? "" : "No-");
1569
1570	return;	/* success */
1571}
1572
1573int main(int argc, char **argv)
1574{
1575	set_base_cpu();
1576
1577	probe_dev_msr();
1578	init_data_structures();
1579
1580	early_cpuid();	/* initial cpuid parse before cmdline */
1581
1582	cmdline(argc, argv);
1583
1584	if (debug)
1585		print_version();
1586
1587	parse_cpuid();
1588
1589	 /* If CPU-set and PKG-set are not initialized, default to all CPUs */
1590	if ((cpu_selected_set == 0) && (pkg_selected_set == 0))
1591		cpu_selected_set = cpu_present_set;
1592
1593	/*
1594	 * If HWP is being enabled, do it now, so that subsequent operations
1595	 * that access HWP registers can work.
1596	 */
1597	if (update_hwp_enable)
1598		for_all_cpus_in_set(cpu_setsize, cpu_selected_set, enable_hwp_on_cpu);
1599
1600	/* If HWP present, but disabled, warn and ignore from here forward */
1601	verify_hwp_is_enabled();
1602
1603	if (req_update_bounds_check())
1604		return -EINVAL;
1605
1606	/* display information only, no updates to settings */
1607	if (!update_epb && !update_turbo && !hwp_update_enabled()) {
1608		if (cpu_selected_set)
1609			for_all_cpus_in_set(cpu_setsize, cpu_selected_set, print_cpu_msrs);
1610
1611		if (has_hwp_request_pkg) {
1612			if (pkg_selected_set == 0)
1613				pkg_selected_set = pkg_present_set;
1614
1615			for_packages(pkg_selected_set, print_pkg_msrs);
1616		}
1617
1618		return 0;
1619	}
1620
1621	/* update CPU set */
1622	if (cpu_selected_set) {
1623		if (update_epb)
1624			for_all_cpus_in_set(cpu_setsize, cpu_selected_set, update_cpu_epb_sysfs);
1625		for_all_cpus_in_set(cpu_setsize, cpu_selected_set, update_sysfs);
1626		for_all_cpus_in_set(cpu_setsize, cpu_selected_set, update_cpu_msrs);
1627
1628	} else if (pkg_selected_set)
1629		for_packages(pkg_selected_set, update_hwp_request_pkg_msr);
1630
1631	return 0;
1632}