FILE
cpumask.h
File preview with syntax highlighting.
linux-6.19.14/include/linux/cpumask.h
1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef __LINUX_CPUMASK_H 3#define __LINUX_CPUMASK_H 4 5/* 6 * Cpumasks provide a bitmap suitable for representing the 7 * set of CPUs in a system, one bit position per CPU number. In general, 8 * only nr_cpu_ids (<= NR_CPUS) bits are valid. 9 */ 10#include <linux/atomic.h> 11#include <linux/bitmap.h> 12#include <linux/cleanup.h> 13#include <linux/cpumask_types.h> 14#include <linux/gfp_types.h> 15#include <linux/numa.h> 16#include <linux/threads.h> 17#include <linux/types.h> 18 19#include <asm/bug.h> 20 21/** 22 * cpumask_pr_args - printf args to output a cpumask 23 * @maskp: cpumask to be printed 24 * 25 * Can be used to provide arguments for '%*pb[l]' when printing a cpumask. 26 */ 27#define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp) 28 29#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) 30#define nr_cpu_ids ((unsigned int)NR_CPUS) 31#else 32extern unsigned int nr_cpu_ids; 33#endif 34 35static __always_inline void set_nr_cpu_ids(unsigned int nr) 36{ 37#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) 38 WARN_ON(nr != nr_cpu_ids); 39#else 40 nr_cpu_ids = nr; 41#endif 42} 43 44/* 45 * We have several different "preferred sizes" for the cpumask 46 * operations, depending on operation. 47 * 48 * For example, the bitmap scanning and operating operations have 49 * optimized routines that work for the single-word case, but only when 50 * the size is constant. So if NR_CPUS fits in one single word, we are 51 * better off using that small constant, in order to trigger the 52 * optimized bit finding. That is 'small_cpumask_size'. 53 * 54 * The clearing and copying operations will similarly perform better 55 * with a constant size, but we limit that size arbitrarily to four 56 * words. We call this 'large_cpumask_size'. 57 * 58 * Finally, some operations just want the exact limit, either because 59 * they set bits or just don't have any faster fixed-sized versions. We 60 * call this just 'nr_cpumask_bits'. 61 * 62 * Note that these optional constants are always guaranteed to be at 63 * least as big as 'nr_cpu_ids' itself is, and all our cpumask 64 * allocations are at least that size (see cpumask_size()). The 65 * optimization comes from being able to potentially use a compile-time 66 * constant instead of a run-time generated exact number of CPUs. 67 */ 68#if NR_CPUS <= BITS_PER_LONG 69 #define small_cpumask_bits ((unsigned int)NR_CPUS) 70 #define large_cpumask_bits ((unsigned int)NR_CPUS) 71#elif NR_CPUS <= 4*BITS_PER_LONG 72 #define small_cpumask_bits nr_cpu_ids 73 #define large_cpumask_bits ((unsigned int)NR_CPUS) 74#else 75 #define small_cpumask_bits nr_cpu_ids 76 #define large_cpumask_bits nr_cpu_ids 77#endif 78#define nr_cpumask_bits nr_cpu_ids 79 80/* 81 * The following particular system cpumasks and operations manage 82 * possible, present, active and online cpus. 83 * 84 * cpu_possible_mask- has bit 'cpu' set iff cpu is populatable 85 * cpu_present_mask - has bit 'cpu' set iff cpu is populated 86 * cpu_enabled_mask - has bit 'cpu' set iff cpu can be brought online 87 * cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler 88 * cpu_active_mask - has bit 'cpu' set iff cpu available to migration 89 * 90 * If !CONFIG_HOTPLUG_CPU, present == possible, and active == online. 91 * 92 * The cpu_possible_mask is fixed at boot time, as the set of CPU IDs 93 * that it is possible might ever be plugged in at anytime during the 94 * life of that system boot. The cpu_present_mask is dynamic(*), 95 * representing which CPUs are currently plugged in. And 96 * cpu_online_mask is the dynamic subset of cpu_present_mask, 97 * indicating those CPUs available for scheduling. 98 * 99 * If HOTPLUG is enabled, then cpu_present_mask varies dynamically, 100 * depending on what ACPI reports as currently plugged in, otherwise 101 * cpu_present_mask is just a copy of cpu_possible_mask. 102 * 103 * (*) Well, cpu_present_mask is dynamic in the hotplug case. If not 104 * hotplug, it's a copy of cpu_possible_mask, hence fixed at boot. 105 * 106 * Subtleties: 107 * 1) UP ARCHes (NR_CPUS == 1, CONFIG_SMP not defined) hardcode 108 * assumption that their single CPU is online. The UP 109 * cpu_{online,possible,present}_masks are placebos. Changing them 110 * will have no useful affect on the following num_*_cpus() 111 * and cpu_*() macros in the UP case. This ugliness is a UP 112 * optimization - don't waste any instructions or memory references 113 * asking if you're online or how many CPUs there are if there is 114 * only one CPU. 115 */ 116 117extern struct cpumask __cpu_possible_mask; 118extern struct cpumask __cpu_online_mask; 119extern struct cpumask __cpu_enabled_mask; 120extern struct cpumask __cpu_present_mask; 121extern struct cpumask __cpu_active_mask; 122extern struct cpumask __cpu_dying_mask; 123#define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask) 124#define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask) 125#define cpu_enabled_mask ((const struct cpumask *)&__cpu_enabled_mask) 126#define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask) 127#define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask) 128#define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask) 129 130extern atomic_t __num_online_cpus; 131extern unsigned int __num_possible_cpus; 132 133extern cpumask_t cpus_booted_once_mask; 134 135static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits) 136{ 137#ifdef CONFIG_DEBUG_PER_CPU_MAPS 138 WARN_ON_ONCE(cpu >= bits); 139#endif /* CONFIG_DEBUG_PER_CPU_MAPS */ 140} 141 142/* verify cpu argument to cpumask_* operators */ 143static __always_inline unsigned int cpumask_check(unsigned int cpu) 144{ 145 cpu_max_bits_warn(cpu, small_cpumask_bits); 146 return cpu; 147} 148 149/** 150 * cpumask_first - get the first cpu in a cpumask 151 * @srcp: the cpumask pointer 152 * 153 * Return: >= nr_cpu_ids if no cpus set. 154 */ 155static __always_inline unsigned int cpumask_first(const struct cpumask *srcp) 156{ 157 return find_first_bit(cpumask_bits(srcp), small_cpumask_bits); 158} 159 160/** 161 * cpumask_first_zero - get the first unset cpu in a cpumask 162 * @srcp: the cpumask pointer 163 * 164 * Return: >= nr_cpu_ids if all cpus are set. 165 */ 166static __always_inline unsigned int cpumask_first_zero(const struct cpumask *srcp) 167{ 168 return find_first_zero_bit(cpumask_bits(srcp), small_cpumask_bits); 169} 170 171/** 172 * cpumask_first_and - return the first cpu from *srcp1 & *srcp2 173 * @srcp1: the first input 174 * @srcp2: the second input 175 * 176 * Return: >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and(). 177 */ 178static __always_inline 179unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2) 180{ 181 return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); 182} 183 184/** 185 * cpumask_first_andnot - return the first cpu from *srcp1 & ~*srcp2 186 * @srcp1: the first input 187 * @srcp2: the second input 188 * 189 * Return: >= nr_cpu_ids if no such cpu found. 190 */ 191static __always_inline 192unsigned int cpumask_first_andnot(const struct cpumask *srcp1, const struct cpumask *srcp2) 193{ 194 return find_first_andnot_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); 195} 196 197/** 198 * cpumask_first_and_and - return the first cpu from *srcp1 & *srcp2 & *srcp3 199 * @srcp1: the first input 200 * @srcp2: the second input 201 * @srcp3: the third input 202 * 203 * Return: >= nr_cpu_ids if no cpus set in all. 204 */ 205static __always_inline 206unsigned int cpumask_first_and_and(const struct cpumask *srcp1, 207 const struct cpumask *srcp2, 208 const struct cpumask *srcp3) 209{ 210 return find_first_and_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), 211 cpumask_bits(srcp3), small_cpumask_bits); 212} 213 214/** 215 * cpumask_last - get the last CPU in a cpumask 216 * @srcp: - the cpumask pointer 217 * 218 * Return: >= nr_cpumask_bits if no CPUs set. 219 */ 220static __always_inline unsigned int cpumask_last(const struct cpumask *srcp) 221{ 222 return find_last_bit(cpumask_bits(srcp), small_cpumask_bits); 223} 224 225/** 226 * cpumask_next - get the next cpu in a cpumask 227 * @n: the cpu prior to the place to search (i.e. return will be > @n) 228 * @srcp: the cpumask pointer 229 * 230 * Return: >= nr_cpu_ids if no further cpus set. 231 */ 232static __always_inline 233unsigned int cpumask_next(int n, const struct cpumask *srcp) 234{ 235 /* -1 is a legal arg here. */ 236 if (n != -1) 237 cpumask_check(n); 238 return find_next_bit(cpumask_bits(srcp), small_cpumask_bits, n + 1); 239} 240 241/** 242 * cpumask_next_zero - get the next unset cpu in a cpumask 243 * @n: the cpu prior to the place to search (i.e. return will be > @n) 244 * @srcp: the cpumask pointer 245 * 246 * Return: >= nr_cpu_ids if no further cpus unset. 247 */ 248static __always_inline 249unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) 250{ 251 /* -1 is a legal arg here. */ 252 if (n != -1) 253 cpumask_check(n); 254 return find_next_zero_bit(cpumask_bits(srcp), small_cpumask_bits, n+1); 255} 256 257#if NR_CPUS == 1 258/* Uniprocessor: there is only one valid CPU */ 259static __always_inline 260unsigned int cpumask_local_spread(unsigned int i, int node) 261{ 262 return 0; 263} 264 265static __always_inline 266unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, 267 const struct cpumask *src2p) 268{ 269 return cpumask_first_and(src1p, src2p); 270} 271 272static __always_inline 273unsigned int cpumask_any_distribute(const struct cpumask *srcp) 274{ 275 return cpumask_first(srcp); 276} 277#else 278unsigned int cpumask_local_spread(unsigned int i, int node); 279unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, 280 const struct cpumask *src2p); 281unsigned int cpumask_any_distribute(const struct cpumask *srcp); 282#endif /* NR_CPUS */ 283 284/** 285 * cpumask_next_and - get the next cpu in *src1p & *src2p 286 * @n: the cpu prior to the place to search (i.e. return will be > @n) 287 * @src1p: the first cpumask pointer 288 * @src2p: the second cpumask pointer 289 * 290 * Return: >= nr_cpu_ids if no further cpus set in both. 291 */ 292static __always_inline 293unsigned int cpumask_next_and(int n, const struct cpumask *src1p, 294 const struct cpumask *src2p) 295{ 296 /* -1 is a legal arg here. */ 297 if (n != -1) 298 cpumask_check(n); 299 return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p), 300 small_cpumask_bits, n + 1); 301} 302 303/** 304 * cpumask_next_andnot - get the next cpu in *src1p & ~*src2p 305 * @n: the cpu prior to the place to search (i.e. return will be > @n) 306 * @src1p: the first cpumask pointer 307 * @src2p: the second cpumask pointer 308 * 309 * Return: >= nr_cpu_ids if no further cpus set in both. 310 */ 311static __always_inline 312unsigned int cpumask_next_andnot(int n, const struct cpumask *src1p, 313 const struct cpumask *src2p) 314{ 315 /* -1 is a legal arg here. */ 316 if (n != -1) 317 cpumask_check(n); 318 return find_next_andnot_bit(cpumask_bits(src1p), cpumask_bits(src2p), 319 small_cpumask_bits, n + 1); 320} 321 322/** 323 * cpumask_next_and_wrap - get the next cpu in *src1p & *src2p, starting from 324 * @n+1. If nothing found, wrap around and start from 325 * the beginning 326 * @n: the cpu prior to the place to search (i.e. search starts from @n+1) 327 * @src1p: the first cpumask pointer 328 * @src2p: the second cpumask pointer 329 * 330 * Return: next set bit, wrapped if needed, or >= nr_cpu_ids if @src1p & @src2p is empty. 331 */ 332static __always_inline 333unsigned int cpumask_next_and_wrap(int n, const struct cpumask *src1p, 334 const struct cpumask *src2p) 335{ 336 /* -1 is a legal arg here. */ 337 if (n != -1) 338 cpumask_check(n); 339 return find_next_and_bit_wrap(cpumask_bits(src1p), cpumask_bits(src2p), 340 small_cpumask_bits, n + 1); 341} 342 343/** 344 * cpumask_next_wrap - get the next cpu in *src, starting from @n+1. If nothing 345 * found, wrap around and start from the beginning 346 * @n: the cpu prior to the place to search (i.e. search starts from @n+1) 347 * @src: cpumask pointer 348 * 349 * Return: next set bit, wrapped if needed, or >= nr_cpu_ids if @src is empty. 350 */ 351static __always_inline 352unsigned int cpumask_next_wrap(int n, const struct cpumask *src) 353{ 354 /* -1 is a legal arg here. */ 355 if (n != -1) 356 cpumask_check(n); 357 return find_next_bit_wrap(cpumask_bits(src), small_cpumask_bits, n + 1); 358} 359 360/** 361 * cpumask_random - get random cpu in *src. 362 * @src: cpumask pointer 363 * 364 * Return: random set bit, or >= nr_cpu_ids if @src is empty. 365 */ 366static __always_inline 367unsigned int cpumask_random(const struct cpumask *src) 368{ 369 return find_random_bit(cpumask_bits(src), nr_cpu_ids); 370} 371 372/** 373 * for_each_cpu - iterate over every cpu in a mask 374 * @cpu: the (optionally unsigned) integer iterator 375 * @mask: the cpumask pointer 376 * 377 * After the loop, cpu is >= nr_cpu_ids. 378 */ 379#define for_each_cpu(cpu, mask) \ 380 for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits) 381 382/** 383 * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location 384 * @cpu: the (optionally unsigned) integer iterator 385 * @mask: the cpumask pointer 386 * @start: the start location 387 * 388 * The implementation does not assume any bit in @mask is set (including @start). 389 * 390 * After the loop, cpu is >= nr_cpu_ids. 391 */ 392#define for_each_cpu_wrap(cpu, mask, start) \ 393 for_each_set_bit_wrap(cpu, cpumask_bits(mask), small_cpumask_bits, start) 394 395/** 396 * for_each_cpu_and - iterate over every cpu in both masks 397 * @cpu: the (optionally unsigned) integer iterator 398 * @mask1: the first cpumask pointer 399 * @mask2: the second cpumask pointer 400 * 401 * This saves a temporary CPU mask in many places. It is equivalent to: 402 * struct cpumask tmp; 403 * cpumask_and(&tmp, &mask1, &mask2); 404 * for_each_cpu(cpu, &tmp) 405 * ... 406 * 407 * After the loop, cpu is >= nr_cpu_ids. 408 */ 409#define for_each_cpu_and(cpu, mask1, mask2) \ 410 for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits) 411 412/** 413 * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding 414 * those present in another. 415 * @cpu: the (optionally unsigned) integer iterator 416 * @mask1: the first cpumask pointer 417 * @mask2: the second cpumask pointer 418 * 419 * This saves a temporary CPU mask in many places. It is equivalent to: 420 * struct cpumask tmp; 421 * cpumask_andnot(&tmp, &mask1, &mask2); 422 * for_each_cpu(cpu, &tmp) 423 * ... 424 * 425 * After the loop, cpu is >= nr_cpu_ids. 426 */ 427#define for_each_cpu_andnot(cpu, mask1, mask2) \ 428 for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits) 429 430/** 431 * for_each_cpu_or - iterate over every cpu present in either mask 432 * @cpu: the (optionally unsigned) integer iterator 433 * @mask1: the first cpumask pointer 434 * @mask2: the second cpumask pointer 435 * 436 * This saves a temporary CPU mask in many places. It is equivalent to: 437 * struct cpumask tmp; 438 * cpumask_or(&tmp, &mask1, &mask2); 439 * for_each_cpu(cpu, &tmp) 440 * ... 441 * 442 * After the loop, cpu is >= nr_cpu_ids. 443 */ 444#define for_each_cpu_or(cpu, mask1, mask2) \ 445 for_each_or_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits) 446 447/** 448 * for_each_cpu_from - iterate over CPUs present in @mask, from @cpu to the end of @mask. 449 * @cpu: the (optionally unsigned) integer iterator 450 * @mask: the cpumask pointer 451 * 452 * After the loop, cpu is >= nr_cpu_ids. 453 */ 454#define for_each_cpu_from(cpu, mask) \ 455 for_each_set_bit_from(cpu, cpumask_bits(mask), small_cpumask_bits) 456 457/** 458 * cpumask_any_but - return an arbitrary cpu in a cpumask, but not this one. 459 * @mask: the cpumask to search 460 * @cpu: the cpu to ignore. 461 * 462 * Often used to find any cpu but smp_processor_id() in a mask. 463 * If @cpu == -1, the function is equivalent to cpumask_any(). 464 * Return: >= nr_cpu_ids if no cpus set. 465 */ 466static __always_inline 467unsigned int cpumask_any_but(const struct cpumask *mask, int cpu) 468{ 469 unsigned int i; 470 471 /* -1 is a legal arg here. */ 472 if (cpu != -1) 473 cpumask_check(cpu); 474 475 for_each_cpu(i, mask) 476 if (i != cpu) 477 break; 478 return i; 479} 480 481/** 482 * cpumask_any_and_but - pick an arbitrary cpu from *mask1 & *mask2, but not this one. 483 * @mask1: the first input cpumask 484 * @mask2: the second input cpumask 485 * @cpu: the cpu to ignore 486 * 487 * If @cpu == -1, the function is equivalent to cpumask_any_and(). 488 * Returns >= nr_cpu_ids if no cpus set. 489 */ 490static __always_inline 491unsigned int cpumask_any_and_but(const struct cpumask *mask1, 492 const struct cpumask *mask2, 493 int cpu) 494{ 495 unsigned int i; 496 497 /* -1 is a legal arg here. */ 498 if (cpu != -1) 499 cpumask_check(cpu); 500 501 i = cpumask_first_and(mask1, mask2); 502 if (i != cpu) 503 return i; 504 505 return cpumask_next_and(cpu, mask1, mask2); 506} 507 508/** 509 * cpumask_any_andnot_but - pick an arbitrary cpu from *mask1 & ~*mask2, but not this one. 510 * @mask1: the first input cpumask 511 * @mask2: the second input cpumask 512 * @cpu: the cpu to ignore 513 * 514 * If @cpu == -1, the function returns the first matching cpu. 515 * Returns >= nr_cpu_ids if no cpus set. 516 */ 517static __always_inline 518unsigned int cpumask_any_andnot_but(const struct cpumask *mask1, 519 const struct cpumask *mask2, 520 int cpu) 521{ 522 unsigned int i; 523 524 /* -1 is a legal arg here. */ 525 if (cpu != -1) 526 cpumask_check(cpu); 527 528 i = cpumask_first_andnot(mask1, mask2); 529 if (i != cpu) 530 return i; 531 532 return cpumask_next_andnot(cpu, mask1, mask2); 533} 534 535/** 536 * cpumask_nth - get the Nth cpu in a cpumask 537 * @srcp: the cpumask pointer 538 * @cpu: the Nth cpu to find, starting from 0 539 * 540 * Return: >= nr_cpu_ids if such cpu doesn't exist. 541 */ 542static __always_inline 543unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp) 544{ 545 return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu)); 546} 547 548/** 549 * cpumask_nth_and - get the Nth cpu in 2 cpumasks 550 * @srcp1: the cpumask pointer 551 * @srcp2: the cpumask pointer 552 * @cpu: the Nth cpu to find, starting from 0 553 * 554 * Return: >= nr_cpu_ids if such cpu doesn't exist. 555 */ 556static __always_inline 557unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, 558 const struct cpumask *srcp2) 559{ 560 return find_nth_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), 561 small_cpumask_bits, cpumask_check(cpu)); 562} 563 564/** 565 * cpumask_nth_and_andnot - get the Nth cpu set in 1st and 2nd cpumask, and clear in 3rd. 566 * @srcp1: the cpumask pointer 567 * @srcp2: the cpumask pointer 568 * @srcp3: the cpumask pointer 569 * @cpu: the Nth cpu to find, starting from 0 570 * 571 * Return: >= nr_cpu_ids if such cpu doesn't exist. 572 */ 573static __always_inline 574unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp1, 575 const struct cpumask *srcp2, 576 const struct cpumask *srcp3) 577{ 578 return find_nth_and_andnot_bit(cpumask_bits(srcp1), 579 cpumask_bits(srcp2), 580 cpumask_bits(srcp3), 581 small_cpumask_bits, cpumask_check(cpu)); 582} 583 584#define CPU_BITS_NONE \ 585{ \ 586 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \ 587} 588 589#define CPU_BITS_CPU0 \ 590{ \ 591 [0] = 1UL \ 592} 593 594/** 595 * cpumask_set_cpu - set a cpu in a cpumask 596 * @cpu: cpu number (< nr_cpu_ids) 597 * @dstp: the cpumask pointer 598 */ 599static __always_inline 600void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) 601{ 602 set_bit(cpumask_check(cpu), cpumask_bits(dstp)); 603} 604 605static __always_inline 606void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) 607{ 608 __set_bit(cpumask_check(cpu), cpumask_bits(dstp)); 609} 610 611/** 612 * cpumask_clear_cpus - clear cpus in a cpumask 613 * @dstp: the cpumask pointer 614 * @cpu: cpu number (< nr_cpu_ids) 615 * @ncpus: number of cpus to clear (< nr_cpu_ids) 616 */ 617static __always_inline void cpumask_clear_cpus(struct cpumask *dstp, 618 unsigned int cpu, unsigned int ncpus) 619{ 620 cpumask_check(cpu + ncpus - 1); 621 bitmap_clear(cpumask_bits(dstp), cpumask_check(cpu), ncpus); 622} 623 624/** 625 * cpumask_clear_cpu - clear a cpu in a cpumask 626 * @cpu: cpu number (< nr_cpu_ids) 627 * @dstp: the cpumask pointer 628 */ 629static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp) 630{ 631 clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); 632} 633 634static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp) 635{ 636 __clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); 637} 638 639/** 640 * cpumask_test_cpu - test for a cpu in a cpumask 641 * @cpu: cpu number (< nr_cpu_ids) 642 * @cpumask: the cpumask pointer 643 * 644 * Return: true if @cpu is set in @cpumask, else returns false 645 */ 646static __always_inline 647bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask) 648{ 649 return test_bit(cpumask_check(cpu), cpumask_bits((cpumask))); 650} 651 652/** 653 * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask 654 * @cpu: cpu number (< nr_cpu_ids) 655 * @cpumask: the cpumask pointer 656 * 657 * test_and_set_bit wrapper for cpumasks. 658 * 659 * Return: true if @cpu is set in old bitmap of @cpumask, else returns false 660 */ 661static __always_inline 662bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) 663{ 664 return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask)); 665} 666 667/** 668 * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask 669 * @cpu: cpu number (< nr_cpu_ids) 670 * @cpumask: the cpumask pointer 671 * 672 * test_and_clear_bit wrapper for cpumasks. 673 * 674 * Return: true if @cpu is set in old bitmap of @cpumask, else returns false 675 */ 676static __always_inline 677bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) 678{ 679 return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask)); 680} 681 682/** 683 * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask 684 * @dstp: the cpumask pointer 685 */ 686static __always_inline void cpumask_setall(struct cpumask *dstp) 687{ 688 if (small_const_nbits(small_cpumask_bits)) { 689 cpumask_bits(dstp)[0] = BITMAP_LAST_WORD_MASK(nr_cpumask_bits); 690 return; 691 } 692 bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits); 693} 694 695/** 696 * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask 697 * @dstp: the cpumask pointer 698 */ 699static __always_inline void cpumask_clear(struct cpumask *dstp) 700{ 701 bitmap_zero(cpumask_bits(dstp), large_cpumask_bits); 702} 703 704/** 705 * cpumask_and - *dstp = *src1p & *src2p 706 * @dstp: the cpumask result 707 * @src1p: the first input 708 * @src2p: the second input 709 * 710 * Return: false if *@dstp is empty, else returns true 711 */ 712static __always_inline 713bool cpumask_and(struct cpumask *dstp, const struct cpumask *src1p, 714 const struct cpumask *src2p) 715{ 716 return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p), 717 cpumask_bits(src2p), small_cpumask_bits); 718} 719 720/** 721 * cpumask_or - *dstp = *src1p | *src2p 722 * @dstp: the cpumask result 723 * @src1p: the first input 724 * @src2p: the second input 725 */ 726static __always_inline 727void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p, 728 const struct cpumask *src2p) 729{ 730 bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p), 731 cpumask_bits(src2p), small_cpumask_bits); 732} 733 734/** 735 * cpumask_weighted_or - *dstp = *src1p | *src2p and return the weight of the result 736 * @dstp: the cpumask result 737 * @src1p: the first input 738 * @src2p: the second input 739 * 740 * Return: The number of bits set in the resulting cpumask @dstp 741 */ 742static __always_inline 743unsigned int cpumask_weighted_or(struct cpumask *dstp, const struct cpumask *src1p, 744 const struct cpumask *src2p) 745{ 746 return bitmap_weighted_or(cpumask_bits(dstp), cpumask_bits(src1p), 747 cpumask_bits(src2p), small_cpumask_bits); 748} 749 750/** 751 * cpumask_xor - *dstp = *src1p ^ *src2p 752 * @dstp: the cpumask result 753 * @src1p: the first input 754 * @src2p: the second input 755 */ 756static __always_inline 757void cpumask_xor(struct cpumask *dstp, const struct cpumask *src1p, 758 const struct cpumask *src2p) 759{ 760 bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p), 761 cpumask_bits(src2p), small_cpumask_bits); 762} 763 764/** 765 * cpumask_andnot - *dstp = *src1p & ~*src2p 766 * @dstp: the cpumask result 767 * @src1p: the first input 768 * @src2p: the second input 769 * 770 * Return: false if *@dstp is empty, else returns true 771 */ 772static __always_inline 773bool cpumask_andnot(struct cpumask *dstp, const struct cpumask *src1p, 774 const struct cpumask *src2p) 775{ 776 return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p), 777 cpumask_bits(src2p), small_cpumask_bits); 778} 779 780/** 781 * cpumask_equal - *src1p == *src2p 782 * @src1p: the first input 783 * @src2p: the second input 784 * 785 * Return: true if the cpumasks are equal, false if not 786 */ 787static __always_inline 788bool cpumask_equal(const struct cpumask *src1p, const struct cpumask *src2p) 789{ 790 return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p), 791 small_cpumask_bits); 792} 793 794/** 795 * cpumask_or_equal - *src1p | *src2p == *src3p 796 * @src1p: the first input 797 * @src2p: the second input 798 * @src3p: the third input 799 * 800 * Return: true if first cpumask ORed with second cpumask == third cpumask, 801 * otherwise false 802 */ 803static __always_inline 804bool cpumask_or_equal(const struct cpumask *src1p, const struct cpumask *src2p, 805 const struct cpumask *src3p) 806{ 807 return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p), 808 cpumask_bits(src3p), small_cpumask_bits); 809} 810 811/** 812 * cpumask_intersects - (*src1p & *src2p) != 0 813 * @src1p: the first input 814 * @src2p: the second input 815 * 816 * Return: true if first cpumask ANDed with second cpumask is non-empty, 817 * otherwise false 818 */ 819static __always_inline 820bool cpumask_intersects(const struct cpumask *src1p, const struct cpumask *src2p) 821{ 822 return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p), 823 small_cpumask_bits); 824} 825 826/** 827 * cpumask_subset - (*src1p & ~*src2p) == 0 828 * @src1p: the first input 829 * @src2p: the second input 830 * 831 * Return: true if *@src1p is a subset of *@src2p, else returns false 832 */ 833static __always_inline 834bool cpumask_subset(const struct cpumask *src1p, const struct cpumask *src2p) 835{ 836 return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p), 837 small_cpumask_bits); 838} 839 840/** 841 * cpumask_empty - *srcp == 0 842 * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear. 843 * 844 * Return: true if srcp is empty (has no bits set), else false 845 */ 846static __always_inline bool cpumask_empty(const struct cpumask *srcp) 847{ 848 return bitmap_empty(cpumask_bits(srcp), small_cpumask_bits); 849} 850 851/** 852 * cpumask_full - *srcp == 0xFFFFFFFF... 853 * @srcp: the cpumask to that all cpus < nr_cpu_ids are set. 854 * 855 * Return: true if srcp is full (has all bits set), else false 856 */ 857static __always_inline bool cpumask_full(const struct cpumask *srcp) 858{ 859 return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits); 860} 861 862/** 863 * cpumask_weight - Count of bits in *srcp 864 * @srcp: the cpumask to count bits (< nr_cpu_ids) in. 865 * 866 * Return: count of bits set in *srcp 867 */ 868static __always_inline unsigned int cpumask_weight(const struct cpumask *srcp) 869{ 870 return bitmap_weight(cpumask_bits(srcp), small_cpumask_bits); 871} 872 873/** 874 * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2) 875 * @srcp1: the cpumask to count bits (< nr_cpu_ids) in. 876 * @srcp2: the cpumask to count bits (< nr_cpu_ids) in. 877 * 878 * Return: count of bits set in both *srcp1 and *srcp2 879 */ 880static __always_inline 881unsigned int cpumask_weight_and(const struct cpumask *srcp1, const struct cpumask *srcp2) 882{ 883 return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); 884} 885 886/** 887 * cpumask_weight_andnot - Count of bits in (*srcp1 & ~*srcp2) 888 * @srcp1: the cpumask to count bits (< nr_cpu_ids) in. 889 * @srcp2: the cpumask to count bits (< nr_cpu_ids) in. 890 * 891 * Return: count of bits set in both *srcp1 and *srcp2 892 */ 893static __always_inline 894unsigned int cpumask_weight_andnot(const struct cpumask *srcp1, 895 const struct cpumask *srcp2) 896{ 897 return bitmap_weight_andnot(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); 898} 899 900/** 901 * cpumask_shift_right - *dstp = *srcp >> n 902 * @dstp: the cpumask result 903 * @srcp: the input to shift 904 * @n: the number of bits to shift by 905 */ 906static __always_inline 907void cpumask_shift_right(struct cpumask *dstp, const struct cpumask *srcp, int n) 908{ 909 bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n, 910 small_cpumask_bits); 911} 912 913/** 914 * cpumask_shift_left - *dstp = *srcp << n 915 * @dstp: the cpumask result 916 * @srcp: the input to shift 917 * @n: the number of bits to shift by 918 */ 919static __always_inline 920void cpumask_shift_left(struct cpumask *dstp, const struct cpumask *srcp, int n) 921{ 922 bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n, 923 nr_cpumask_bits); 924} 925 926/** 927 * cpumask_copy - *dstp = *srcp 928 * @dstp: the result 929 * @srcp: the input cpumask 930 */ 931static __always_inline 932void cpumask_copy(struct cpumask *dstp, const struct cpumask *srcp) 933{ 934 bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), large_cpumask_bits); 935} 936 937/** 938 * cpumask_any - pick an arbitrary cpu from *srcp 939 * @srcp: the input cpumask 940 * 941 * Return: >= nr_cpu_ids if no cpus set. 942 */ 943#define cpumask_any(srcp) cpumask_first(srcp) 944 945/** 946 * cpumask_any_and - pick an arbitrary cpu from *mask1 & *mask2 947 * @mask1: the first input cpumask 948 * @mask2: the second input cpumask 949 * 950 * Return: >= nr_cpu_ids if no cpus set. 951 */ 952#define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2)) 953 954/** 955 * cpumask_of - the cpumask containing just a given cpu 956 * @cpu: the cpu (<= nr_cpu_ids) 957 */ 958#define cpumask_of(cpu) (get_cpu_mask(cpu)) 959 960/** 961 * cpumask_parse_user - extract a cpumask from a user string 962 * @buf: the buffer to extract from 963 * @len: the length of the buffer 964 * @dstp: the cpumask to set. 965 * 966 * Return: -errno, or 0 for success. 967 */ 968static __always_inline 969int cpumask_parse_user(const char __user *buf, int len, struct cpumask *dstp) 970{ 971 return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits); 972} 973 974/** 975 * cpumask_parselist_user - extract a cpumask from a user string 976 * @buf: the buffer to extract from 977 * @len: the length of the buffer 978 * @dstp: the cpumask to set. 979 * 980 * Return: -errno, or 0 for success. 981 */ 982static __always_inline 983int cpumask_parselist_user(const char __user *buf, int len, struct cpumask *dstp) 984{ 985 return bitmap_parselist_user(buf, len, cpumask_bits(dstp), 986 nr_cpumask_bits); 987} 988 989/** 990 * cpumask_parse - extract a cpumask from a string 991 * @buf: the buffer to extract from 992 * @dstp: the cpumask to set. 993 * 994 * Return: -errno, or 0 for success. 995 */ 996static __always_inline int cpumask_parse(const char *buf, struct cpumask *dstp) 997{ 998 return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits); 999} 1000 1001/** 1002 * cpulist_parse - extract a cpumask from a user string of ranges 1003 * @buf: the buffer to extract from 1004 * @dstp: the cpumask to set. 1005 * 1006 * Return: -errno, or 0 for success. 1007 */ 1008static __always_inline int cpulist_parse(const char *buf, struct cpumask *dstp) 1009{ 1010 return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits); 1011} 1012 1013/** 1014 * cpumask_size - calculate size to allocate for a 'struct cpumask' in bytes 1015 * 1016 * Return: size to allocate for a &struct cpumask in bytes 1017 */ 1018static __always_inline unsigned int cpumask_size(void) 1019{ 1020 return bitmap_size(large_cpumask_bits); 1021} 1022 1023#ifdef CONFIG_CPUMASK_OFFSTACK 1024 1025#define this_cpu_cpumask_var_ptr(x) this_cpu_read(x) 1026#define __cpumask_var_read_mostly __read_mostly 1027#define CPUMASK_VAR_NULL NULL 1028 1029bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node); 1030 1031static __always_inline 1032bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) 1033{ 1034 return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node); 1035} 1036 1037/** 1038 * alloc_cpumask_var - allocate a struct cpumask 1039 * @mask: pointer to cpumask_var_t where the cpumask is returned 1040 * @flags: GFP_ flags 1041 * 1042 * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is 1043 * a nop returning a constant 1 (in <linux/cpumask.h>). 1044 * 1045 * See alloc_cpumask_var_node. 1046 * 1047 * Return: %true if allocation succeeded, %false if not 1048 */ 1049static __always_inline 1050bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) 1051{ 1052 return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE); 1053} 1054 1055static __always_inline 1056bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) 1057{ 1058 return alloc_cpumask_var(mask, flags | __GFP_ZERO); 1059} 1060 1061void alloc_bootmem_cpumask_var(cpumask_var_t *mask); 1062void free_cpumask_var(cpumask_var_t mask); 1063void free_bootmem_cpumask_var(cpumask_var_t mask); 1064 1065static __always_inline bool cpumask_available(cpumask_var_t mask) 1066{ 1067 return mask != NULL; 1068} 1069 1070#else 1071 1072#define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x) 1073#define __cpumask_var_read_mostly 1074#define CPUMASK_VAR_NULL {} 1075 1076static __always_inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) 1077{ 1078 return true; 1079} 1080 1081static __always_inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, 1082 int node) 1083{ 1084 return true; 1085} 1086 1087static __always_inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) 1088{ 1089 cpumask_clear(*mask); 1090 return true; 1091} 1092 1093static __always_inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, 1094 int node) 1095{ 1096 cpumask_clear(*mask); 1097 return true; 1098} 1099 1100static __always_inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask) 1101{ 1102} 1103 1104static __always_inline void free_cpumask_var(cpumask_var_t mask) 1105{ 1106} 1107 1108static __always_inline void free_bootmem_cpumask_var(cpumask_var_t mask) 1109{ 1110} 1111 1112static __always_inline bool cpumask_available(cpumask_var_t mask) 1113{ 1114 return true; 1115} 1116#endif /* CONFIG_CPUMASK_OFFSTACK */ 1117 1118DEFINE_FREE(free_cpumask_var, struct cpumask *, if (_T) free_cpumask_var(_T)); 1119 1120/* It's common to want to use cpu_all_mask in struct member initializers, 1121 * so it has to refer to an address rather than a pointer. */ 1122extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS); 1123#define cpu_all_mask to_cpumask(cpu_all_bits) 1124 1125/* First bits of cpu_bit_bitmap are in fact unset. */ 1126#define cpu_none_mask to_cpumask(cpu_bit_bitmap[0]) 1127 1128#if NR_CPUS == 1 1129/* Uniprocessor: the possible/online/present masks are always "1" */ 1130#define for_each_possible_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) 1131#define for_each_online_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) 1132#define for_each_present_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) 1133 1134#define for_each_possible_cpu_wrap(cpu, start) \ 1135 for ((void)(start), (cpu) = 0; (cpu) < 1; (cpu)++) 1136#define for_each_online_cpu_wrap(cpu, start) \ 1137 for ((void)(start), (cpu) = 0; (cpu) < 1; (cpu)++) 1138#else 1139#define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask) 1140#define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask) 1141#define for_each_enabled_cpu(cpu) for_each_cpu((cpu), cpu_enabled_mask) 1142#define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask) 1143 1144#define for_each_possible_cpu_wrap(cpu, start) \ 1145 for_each_cpu_wrap((cpu), cpu_possible_mask, (start)) 1146#define for_each_online_cpu_wrap(cpu, start) \ 1147 for_each_cpu_wrap((cpu), cpu_online_mask, (start)) 1148#endif 1149 1150/* Wrappers for arch boot code to manipulate normally-constant masks */ 1151void init_cpu_present(const struct cpumask *src); 1152void init_cpu_possible(const struct cpumask *src); 1153 1154#define assign_cpu(cpu, mask, val) \ 1155 assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val)) 1156 1157#define __assign_cpu(cpu, mask, val) \ 1158 __assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val)) 1159 1160#define set_cpu_enabled(cpu, enabled) assign_cpu((cpu), &__cpu_enabled_mask, (enabled)) 1161#define set_cpu_present(cpu, present) assign_cpu((cpu), &__cpu_present_mask, (present)) 1162#define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active)) 1163#define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying)) 1164 1165void set_cpu_online(unsigned int cpu, bool online); 1166void set_cpu_possible(unsigned int cpu, bool possible); 1167 1168/** 1169 * to_cpumask - convert a NR_CPUS bitmap to a struct cpumask * 1170 * @bitmap: the bitmap 1171 * 1172 * There are a few places where cpumask_var_t isn't appropriate and 1173 * static cpumasks must be used (eg. very early boot), yet we don't 1174 * expose the definition of 'struct cpumask'. 1175 * 1176 * This does the conversion, and can be used as a constant initializer. 1177 */ 1178#define to_cpumask(bitmap) \ 1179 ((struct cpumask *)(1 ? (bitmap) \ 1180 : (void *)sizeof(__check_is_bitmap(bitmap)))) 1181 1182static __always_inline int __check_is_bitmap(const unsigned long *bitmap) 1183{ 1184 return 1; 1185} 1186 1187/* 1188 * Special-case data structure for "single bit set only" constant CPU masks. 1189 * 1190 * We pre-generate all the 64 (or 32) possible bit positions, with enough 1191 * padding to the left and the right, and return the constant pointer 1192 * appropriately offset. 1193 */ 1194extern const unsigned long 1195 cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)]; 1196 1197static __always_inline const struct cpumask *get_cpu_mask(unsigned int cpu) 1198{ 1199 const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG]; 1200 p -= cpu / BITS_PER_LONG; 1201 return to_cpumask(p); 1202} 1203 1204#if NR_CPUS > 1 1205/** 1206 * num_online_cpus() - Read the number of online CPUs 1207 * 1208 * Despite the fact that __num_online_cpus is of type atomic_t, this 1209 * interface gives only a momentary snapshot and is not protected against 1210 * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held 1211 * region. 1212 * 1213 * Return: momentary snapshot of the number of online CPUs 1214 */ 1215static __always_inline unsigned int num_online_cpus(void) 1216{ 1217 return raw_atomic_read(&__num_online_cpus); 1218} 1219 1220static __always_inline unsigned int num_possible_cpus(void) 1221{ 1222 return __num_possible_cpus; 1223} 1224 1225#define num_enabled_cpus() cpumask_weight(cpu_enabled_mask) 1226#define num_present_cpus() cpumask_weight(cpu_present_mask) 1227#define num_active_cpus() cpumask_weight(cpu_active_mask) 1228 1229static __always_inline bool cpu_online(unsigned int cpu) 1230{ 1231 return cpumask_test_cpu(cpu, cpu_online_mask); 1232} 1233 1234static __always_inline bool cpu_enabled(unsigned int cpu) 1235{ 1236 return cpumask_test_cpu(cpu, cpu_enabled_mask); 1237} 1238 1239static __always_inline bool cpu_possible(unsigned int cpu) 1240{ 1241 return cpumask_test_cpu(cpu, cpu_possible_mask); 1242} 1243 1244static __always_inline bool cpu_present(unsigned int cpu) 1245{ 1246 return cpumask_test_cpu(cpu, cpu_present_mask); 1247} 1248 1249static __always_inline bool cpu_active(unsigned int cpu) 1250{ 1251 return cpumask_test_cpu(cpu, cpu_active_mask); 1252} 1253 1254static __always_inline bool cpu_dying(unsigned int cpu) 1255{ 1256 return cpumask_test_cpu(cpu, cpu_dying_mask); 1257} 1258 1259#else 1260 1261#define num_online_cpus() 1U 1262#define num_possible_cpus() 1U 1263#define num_enabled_cpus() 1U 1264#define num_present_cpus() 1U 1265#define num_active_cpus() 1U 1266 1267static __always_inline bool cpu_online(unsigned int cpu) 1268{ 1269 return cpu == 0; 1270} 1271 1272static __always_inline bool cpu_possible(unsigned int cpu) 1273{ 1274 return cpu == 0; 1275} 1276 1277static __always_inline bool cpu_enabled(unsigned int cpu) 1278{ 1279 return cpu == 0; 1280} 1281 1282static __always_inline bool cpu_present(unsigned int cpu) 1283{ 1284 return cpu == 0; 1285} 1286 1287static __always_inline bool cpu_active(unsigned int cpu) 1288{ 1289 return cpu == 0; 1290} 1291 1292static __always_inline bool cpu_dying(unsigned int cpu) 1293{ 1294 return false; 1295} 1296 1297#endif /* NR_CPUS > 1 */ 1298 1299#define cpu_is_offline(cpu) unlikely(!cpu_online(cpu)) 1300 1301#if NR_CPUS <= BITS_PER_LONG 1302#define CPU_BITS_ALL \ 1303{ \ 1304 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \ 1305} 1306 1307#else /* NR_CPUS > BITS_PER_LONG */ 1308 1309#define CPU_BITS_ALL \ 1310{ \ 1311 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \ 1312 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \ 1313} 1314#endif /* NR_CPUS > BITS_PER_LONG */ 1315 1316/** 1317 * cpumap_print_to_pagebuf - copies the cpumask into the buffer either 1318 * as comma-separated list of cpus or hex values of cpumask 1319 * @list: indicates whether the cpumap must be list 1320 * @mask: the cpumask to copy 1321 * @buf: the buffer to copy into 1322 * 1323 * Return: the length of the (null-terminated) @buf string, zero if 1324 * nothing is copied. 1325 */ 1326static __always_inline ssize_t 1327cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask) 1328{ 1329 return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask), 1330 nr_cpu_ids); 1331} 1332 1333/** 1334 * cpumap_print_bitmask_to_buf - copies the cpumask into the buffer as 1335 * hex values of cpumask 1336 * 1337 * @buf: the buffer to copy into 1338 * @mask: the cpumask to copy 1339 * @off: in the string from which we are copying, we copy to @buf 1340 * @count: the maximum number of bytes to print 1341 * 1342 * The function prints the cpumask into the buffer as hex values of 1343 * cpumask; Typically used by bin_attribute to export cpumask bitmask 1344 * ABI. 1345 * 1346 * Return: the length of how many bytes have been copied, excluding 1347 * terminating '\0'. 1348 */ 1349static __always_inline 1350ssize_t cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask, 1351 loff_t off, size_t count) 1352{ 1353 return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask), 1354 nr_cpu_ids, off, count) - 1; 1355} 1356 1357/** 1358 * cpumap_print_list_to_buf - copies the cpumask into the buffer as 1359 * comma-separated list of cpus 1360 * @buf: the buffer to copy into 1361 * @mask: the cpumask to copy 1362 * @off: in the string from which we are copying, we copy to @buf 1363 * @count: the maximum number of bytes to print 1364 * 1365 * Everything is same with the above cpumap_print_bitmask_to_buf() 1366 * except the print format. 1367 * 1368 * Return: the length of how many bytes have been copied, excluding 1369 * terminating '\0'. 1370 */ 1371static __always_inline 1372ssize_t cpumap_print_list_to_buf(char *buf, const struct cpumask *mask, 1373 loff_t off, size_t count) 1374{ 1375 return bitmap_print_list_to_buf(buf, cpumask_bits(mask), 1376 nr_cpu_ids, off, count) - 1; 1377} 1378 1379#if NR_CPUS <= BITS_PER_LONG 1380#define CPU_MASK_ALL \ 1381(cpumask_t) { { \ 1382 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \ 1383} } 1384#else 1385#define CPU_MASK_ALL \ 1386(cpumask_t) { { \ 1387 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \ 1388 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \ 1389} } 1390#endif /* NR_CPUS > BITS_PER_LONG */ 1391 1392#define CPU_MASK_NONE \ 1393(cpumask_t) { { \ 1394 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \ 1395} } 1396 1397#define CPU_MASK_CPU0 \ 1398(cpumask_t) { { \ 1399 [0] = 1UL \ 1400} } 1401 1402/* 1403 * Provide a valid theoretical max size for cpumap and cpulist sysfs files 1404 * to avoid breaking userspace which may allocate a buffer based on the size 1405 * reported by e.g. fstat. 1406 * 1407 * for cpumap NR_CPUS * 9/32 - 1 should be an exact length. 1408 * 1409 * For cpulist 7 is (ceil(log10(NR_CPUS)) + 1) allowing for NR_CPUS to be up 1410 * to 2 orders of magnitude larger than 8192. And then we divide by 2 to 1411 * cover a worst-case of every other cpu being on one of two nodes for a 1412 * very large NR_CPUS. 1413 * 1414 * Use PAGE_SIZE as a minimum for smaller configurations while avoiding 1415 * unsigned comparison to -1. 1416 */ 1417#define CPUMAP_FILE_MAX_BYTES (((NR_CPUS * 9)/32 > PAGE_SIZE) \ 1418 ? (NR_CPUS * 9)/32 - 1 : PAGE_SIZE) 1419#define CPULIST_FILE_MAX_BYTES (((NR_CPUS * 7)/2 > PAGE_SIZE) ? (NR_CPUS * 7)/2 : PAGE_SIZE) 1420 1421#endif /* __LINUX_CPUMASK_H */
~/
Read-only file preview.