main.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *  linux/init/main.c
   4 *
   5 *  Copyright (C) 1991, 1992  Linus Torvalds
   6 *
   7 *  GK 2/5/95  -  Changed to support mounting root fs via NFS
   8 *  Added initrd & change_root: Werner Almesberger & Hans Lermen, Feb '96
   9 *  Moan early if gcc is old, avoiding bogus kernels - Paul Gortmaker, May '96
  10 *  Simplified starting of init:  Michael A. Griffith <grif@acm.org>
  11 */
  12
  13#define DEBUG		/* Enable initcall_debug */
  14
  15#include <linux/types.h>
  16#include <linux/export.h>
  17#include <linux/extable.h>
  18#include <linux/module.h>
  19#include <linux/proc_fs.h>
  20#include <linux/binfmts.h>
  21#include <linux/kernel.h>
  22#include <linux/syscalls.h>
  23#include <linux/stackprotector.h>
  24#include <linux/string.h>
  25#include <linux/ctype.h>
  26#include <linux/delay.h>
  27#include <linux/ioport.h>
  28#include <linux/init.h>
  29#include <linux/initrd.h>
  30#include <linux/memblock.h>
  31#include <linux/acpi.h>
  32#include <linux/bootconfig.h>
  33#include <linux/console.h>
  34#include <linux/nmi.h>
  35#include <linux/percpu.h>
  36#include <linux/kmod.h>
  37#include <linux/kprobes.h>
  38#include <linux/kmsan.h>
  39#include <linux/vmalloc.h>
  40#include <linux/kernel_stat.h>
  41#include <linux/start_kernel.h>
  42#include <linux/security.h>
  43#include <linux/smp.h>
  44#include <linux/profile.h>
  45#include <linux/kfence.h>
  46#include <linux/rcupdate.h>
  47#include <linux/srcu.h>
  48#include <linux/moduleparam.h>
  49#include <linux/kallsyms.h>
  50#include <linux/buildid.h>
  51#include <linux/writeback.h>
  52#include <linux/cpu.h>
  53#include <linux/cpuset.h>
  54#include <linux/memcontrol.h>
  55#include <linux/cgroup.h>
  56#include <linux/tick.h>
  57#include <linux/sched/isolation.h>
  58#include <linux/interrupt.h>
  59#include <linux/taskstats_kern.h>
  60#include <linux/delayacct.h>
  61#include <linux/unistd.h>
  62#include <linux/utsname.h>
  63#include <linux/rmap.h>
  64#include <linux/mempolicy.h>
  65#include <linux/key.h>
  66#include <linux/debug_locks.h>
  67#include <linux/debugobjects.h>
  68#include <linux/lockdep.h>
  69#include <linux/kmemleak.h>
  70#include <linux/padata.h>
  71#include <linux/pid_namespace.h>
  72#include <linux/device/driver.h>
  73#include <linux/kthread.h>
  74#include <linux/sched.h>
  75#include <linux/sched/init.h>
  76#include <linux/signal.h>
  77#include <linux/idr.h>
  78#include <linux/kgdb.h>
  79#include <linux/ftrace.h>
  80#include <linux/async.h>
  81#include <linux/shmem_fs.h>
  82#include <linux/slab.h>
  83#include <linux/perf_event.h>
  84#include <linux/ptrace.h>
  85#include <linux/pti.h>
  86#include <linux/blkdev.h>
  87#include <linux/sched/clock.h>
  88#include <linux/sched/task.h>
  89#include <linux/sched/task_stack.h>
  90#include <linux/context_tracking.h>
  91#include <linux/random.h>
  92#include <linux/moduleloader.h>
  93#include <linux/list.h>
  94#include <linux/integrity.h>
  95#include <linux/proc_ns.h>
  96#include <linux/io.h>
  97#include <linux/cache.h>
  98#include <linux/rodata_test.h>
  99#include <linux/jump_label.h>
 100#include <linux/kcsan.h>
 101#include <linux/init_syscalls.h>
 102#include <linux/stackdepot.h>
 103#include <linux/randomize_kstack.h>
 104#include <linux/pidfs.h>
 105#include <linux/ptdump.h>
 106#include <linux/time_namespace.h>
 107#include <net/net_namespace.h>
 108
 109#include <asm/io.h>
 110#include <asm/setup.h>
 111#include <asm/sections.h>
 112#include <asm/cacheflush.h>
 113
 114#define CREATE_TRACE_POINTS
 115#include <trace/events/initcall.h>
 116
 117#include <kunit/test.h>
 118
 119static int kernel_init(void *);
 120
 121/*
 122 * Debug helper: via this flag we know that we are in 'early bootup code'
 123 * where only the boot processor is running with IRQ disabled.  This means
 124 * two things - IRQ must not be enabled before the flag is cleared and some
 125 * operations which are not allowed with IRQ disabled are allowed while the
 126 * flag is set.
 127 */
 128bool early_boot_irqs_disabled __read_mostly;
 129
 130enum system_states system_state __read_mostly;
 131EXPORT_SYMBOL(system_state);
 132
 133/*
 134 * Boot command-line arguments
 135 */
 136#define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMIT
 137#define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMIT
 138
 139/* Default late time init is NULL. archs can override this later. */
 140void (*__initdata late_time_init)(void);
 141
 142/* Untouched command line saved by arch-specific code. */
 143char __initdata boot_command_line[COMMAND_LINE_SIZE];
 144/* Untouched saved command line (eg. for /proc) */
 145char *saved_command_line __ro_after_init;
 146unsigned int saved_command_line_len __ro_after_init;
 147/* Command line for parameter parsing */
 148static char *static_command_line;
 149/* Untouched extra command line */
 150static char *extra_command_line;
 151/* Extra init arguments */
 152static char *extra_init_args;
 153
 154#ifdef CONFIG_BOOT_CONFIG
 155/* Is bootconfig on command line? */
 156static bool bootconfig_found;
 157static size_t initargs_offs;
 158#else
 159# define bootconfig_found false
 160# define initargs_offs 0
 161#endif
 162
 163static char *execute_command;
 164static char *ramdisk_execute_command = "/init";
 165
 166/*
 167 * Used to generate warnings if static_key manipulation functions are used
 168 * before jump_label_init is called.
 169 */
 170bool static_key_initialized __read_mostly;
 171EXPORT_SYMBOL_GPL(static_key_initialized);
 172
 173/*
 174 * If set, this is an indication to the drivers that reset the underlying
 175 * device before going ahead with the initialization otherwise driver might
 176 * rely on the BIOS and skip the reset operation.
 177 *
 178 * This is useful if kernel is booting in an unreliable environment.
 179 * For ex. kdump situation where previous kernel has crashed, BIOS has been
 180 * skipped and devices will be in unknown state.
 181 */
 182unsigned int reset_devices;
 183EXPORT_SYMBOL(reset_devices);
 184
 185static int __init set_reset_devices(char *str)
 186{
 187	reset_devices = 1;
 188	return 1;
 189}
 190
 191__setup("reset_devices", set_reset_devices);
 192
 193static const char *argv_init[MAX_INIT_ARGS+2] = { "init", NULL, };
 194const char *envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, };
 195static const char *panic_later, *panic_param;
 196
 197static bool __init obsolete_checksetup(char *line)
 198{
 199	const struct obs_kernel_param *p;
 200	bool had_early_param = false;
 201
 202	p = __setup_start;
 203	do {
 204		int n = strlen(p->str);
 205		if (parameqn(line, p->str, n)) {
 206			if (p->early) {
 207				/* Already done in parse_early_param?
 208				 * (Needs exact match on param part).
 209				 * Keep iterating, as we can have early
 210				 * params and __setups of same names 8( */
 211				if (line[n] == '\0' || line[n] == '=')
 212					had_early_param = true;
 213			} else if (!p->setup_func) {
 214				pr_warn("Parameter %s is obsolete, ignored\n",
 215					p->str);
 216				return true;
 217			} else if (p->setup_func(line + n))
 218				return true;
 219		}
 220		p++;
 221	} while (p < __setup_end);
 222
 223	return had_early_param;
 224}
 225
 226/*
 227 * This should be approx 2 Bo*oMips to start (note initial shift), and will
 228 * still work even if initially too large, it will just take slightly longer
 229 */
 230unsigned long loops_per_jiffy = (1<<12);
 231EXPORT_SYMBOL(loops_per_jiffy);
 232
 233static int __init debug_kernel(char *str)
 234{
 235	console_loglevel = CONSOLE_LOGLEVEL_DEBUG;
 236	return 0;
 237}
 238
 239static int __init quiet_kernel(char *str)
 240{
 241	console_loglevel = CONSOLE_LOGLEVEL_QUIET;
 242	return 0;
 243}
 244
 245early_param("debug", debug_kernel);
 246early_param("quiet", quiet_kernel);
 247
 248static int __init loglevel(char *str)
 249{
 250	int newlevel;
 251
 252	/*
 253	 * Only update loglevel value when a correct setting was passed,
 254	 * to prevent blind crashes (when loglevel being set to 0) that
 255	 * are quite hard to debug
 256	 */
 257	if (get_option(&str, &newlevel)) {
 258		console_loglevel = newlevel;
 259		return 0;
 260	}
 261
 262	return -EINVAL;
 263}
 264
 265early_param("loglevel", loglevel);
 266
 267#ifdef CONFIG_BLK_DEV_INITRD
 268static void * __init get_boot_config_from_initrd(size_t *_size)
 269{
 270	u32 size, csum;
 271	char *data;
 272	u32 *hdr;
 273	int i;
 274
 275	if (!initrd_end)
 276		return NULL;
 277
 278	data = (char *)initrd_end - BOOTCONFIG_MAGIC_LEN;
 279	/*
 280	 * Since Grub may align the size of initrd to 4, we must
 281	 * check the preceding 3 bytes as well.
 282	 */
 283	for (i = 0; i < 4; i++) {
 284		if (!memcmp(data, BOOTCONFIG_MAGIC, BOOTCONFIG_MAGIC_LEN))
 285			goto found;
 286		data--;
 287	}
 288	return NULL;
 289
 290found:
 291	hdr = (u32 *)(data - 8);
 292	size = le32_to_cpu(hdr[0]);
 293	csum = le32_to_cpu(hdr[1]);
 294
 295	data = ((void *)hdr) - size;
 296	if ((unsigned long)data < initrd_start) {
 297		pr_err("bootconfig size %d is greater than initrd size %ld\n",
 298			size, initrd_end - initrd_start);
 299		return NULL;
 300	}
 301
 302	if (xbc_calc_checksum(data, size) != csum) {
 303		pr_err("bootconfig checksum failed\n");
 304		return NULL;
 305	}
 306
 307	/* Remove bootconfig from initramfs/initrd */
 308	initrd_end = (unsigned long)data;
 309	if (_size)
 310		*_size = size;
 311
 312	return data;
 313}
 314#else
 315static void * __init get_boot_config_from_initrd(size_t *_size)
 316{
 317	return NULL;
 318}
 319#endif
 320
 321#ifdef CONFIG_BOOT_CONFIG
 322
 323static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata;
 324
 325#define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0)
 326
 327static int __init xbc_snprint_cmdline(char *buf, size_t size,
 328				      struct xbc_node *root)
 329{
 330	struct xbc_node *knode, *vnode;
 331	char *end = buf + size;
 332	const char *val, *q;
 333	int ret;
 334
 335	xbc_node_for_each_key_value(root, knode, val) {
 336		ret = xbc_node_compose_key_after(root, knode,
 337					xbc_namebuf, XBC_KEYLEN_MAX);
 338		if (ret < 0)
 339			return ret;
 340
 341		vnode = xbc_node_get_child(knode);
 342		if (!vnode) {
 343			ret = snprintf(buf, rest(buf, end), "%s ", xbc_namebuf);
 344			if (ret < 0)
 345				return ret;
 346			buf += ret;
 347			continue;
 348		}
 349		xbc_array_for_each_value(vnode, val) {
 350			/*
 351			 * For prettier and more readable /proc/cmdline, only
 352			 * quote the value when necessary, i.e. when it contains
 353			 * whitespace.
 354			 */
 355			q = strpbrk(val, " \t\r\n") ? "\"" : "";
 356			ret = snprintf(buf, rest(buf, end), "%s=%s%s%s ",
 357				       xbc_namebuf, q, val, q);
 358			if (ret < 0)
 359				return ret;
 360			buf += ret;
 361		}
 362	}
 363
 364	return buf - (end - size);
 365}
 366#undef rest
 367
 368/* Make an extra command line under given key word */
 369static char * __init xbc_make_cmdline(const char *key)
 370{
 371	struct xbc_node *root;
 372	char *new_cmdline;
 373	int ret, len = 0;
 374
 375	root = xbc_find_node(key);
 376	if (!root)
 377		return NULL;
 378
 379	/* Count required buffer size */
 380	len = xbc_snprint_cmdline(NULL, 0, root);
 381	if (len <= 0)
 382		return NULL;
 383
 384	new_cmdline = memblock_alloc(len + 1, SMP_CACHE_BYTES);
 385	if (!new_cmdline) {
 386		pr_err("Failed to allocate memory for extra kernel cmdline.\n");
 387		return NULL;
 388	}
 389
 390	ret = xbc_snprint_cmdline(new_cmdline, len + 1, root);
 391	if (ret < 0 || ret > len) {
 392		pr_err("Failed to print extra kernel cmdline.\n");
 393		memblock_free(new_cmdline, len + 1);
 394		return NULL;
 395	}
 396
 397	return new_cmdline;
 398}
 399
 400static int __init bootconfig_params(char *param, char *val,
 401				    const char *unused, void *arg)
 402{
 403	if (strcmp(param, "bootconfig") == 0) {
 404		bootconfig_found = true;
 405	}
 406	return 0;
 407}
 408
 409static int __init warn_bootconfig(char *str)
 410{
 411	/* The 'bootconfig' has been handled by bootconfig_params(). */
 412	return 0;
 413}
 414
 415static void __init setup_boot_config(void)
 416{
 417	static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;
 418	const char *msg, *data;
 419	int pos, ret;
 420	size_t size;
 421	char *err;
 422
 423	/* Cut out the bootconfig data even if we have no bootconfig option */
 424	data = get_boot_config_from_initrd(&size);
 425	/* If there is no bootconfig in initrd, try embedded one. */
 426	if (!data)
 427		data = xbc_get_embedded_bootconfig(&size);
 428
 429	strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
 430	err = parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0, NULL,
 431			 bootconfig_params);
 432
 433	if (IS_ERR(err) || !(bootconfig_found || IS_ENABLED(CONFIG_BOOT_CONFIG_FORCE)))
 434		return;
 435
 436	/* parse_args() stops at the next param of '--' and returns an address */
 437	if (err)
 438		initargs_offs = err - tmp_cmdline;
 439
 440	if (!data) {
 441		/* If user intended to use bootconfig, show an error level message */
 442		if (bootconfig_found)
 443			pr_err("'bootconfig' found on command line, but no bootconfig found\n");
 444		else
 445			pr_info("No bootconfig data provided, so skipping bootconfig");
 446		return;
 447	}
 448
 449	if (size >= XBC_DATA_MAX) {
 450		pr_err("bootconfig size %ld greater than max size %d\n",
 451			(long)size, XBC_DATA_MAX);
 452		return;
 453	}
 454
 455	ret = xbc_init(data, size, &msg, &pos);
 456	if (ret < 0) {
 457		if (pos < 0)
 458			pr_err("Failed to init bootconfig: %s.\n", msg);
 459		else
 460			pr_err("Failed to parse bootconfig: %s at %d.\n",
 461				msg, pos);
 462	} else {
 463		xbc_get_info(&ret, NULL);
 464		pr_info("Load bootconfig: %ld bytes %d nodes\n", (long)size, ret);
 465		/* keys starting with "kernel." are passed via cmdline */
 466		extra_command_line = xbc_make_cmdline("kernel");
 467		/* Also, "init." keys are init arguments */
 468		extra_init_args = xbc_make_cmdline("init");
 469	}
 470	return;
 471}
 472
 473static void __init exit_boot_config(void)
 474{
 475	xbc_exit();
 476}
 477
 478#else	/* !CONFIG_BOOT_CONFIG */
 479
 480static void __init setup_boot_config(void)
 481{
 482	/* Remove bootconfig data from initrd */
 483	get_boot_config_from_initrd(NULL);
 484}
 485
 486static int __init warn_bootconfig(char *str)
 487{
 488	pr_warn("WARNING: 'bootconfig' found on the kernel command line but CONFIG_BOOT_CONFIG is not set.\n");
 489	return 0;
 490}
 491
 492#define exit_boot_config()	do {} while (0)
 493
 494#endif	/* CONFIG_BOOT_CONFIG */
 495
 496early_param("bootconfig", warn_bootconfig);
 497
 498bool __init cmdline_has_extra_options(void)
 499{
 500	return extra_command_line || extra_init_args;
 501}
 502
 503/* Change NUL term back to "=", to make "param" the whole string. */
 504static void __init repair_env_string(char *param, char *val)
 505{
 506	if (val) {
 507		/* param=val or param="val"? */
 508		if (val == param+strlen(param)+1)
 509			val[-1] = '=';
 510		else if (val == param+strlen(param)+2) {
 511			val[-2] = '=';
 512			memmove(val-1, val, strlen(val)+1);
 513		} else
 514			BUG();
 515	}
 516}
 517
 518/* Anything after -- gets handed straight to init. */
 519static int __init set_init_arg(char *param, char *val,
 520			       const char *unused, void *arg)
 521{
 522	unsigned int i;
 523
 524	if (panic_later)
 525		return 0;
 526
 527	repair_env_string(param, val);
 528
 529	for (i = 0; argv_init[i]; i++) {
 530		if (i == MAX_INIT_ARGS) {
 531			panic_later = "init";
 532			panic_param = param;
 533			return 0;
 534		}
 535	}
 536	argv_init[i] = param;
 537	return 0;
 538}
 539
 540/*
 541 * Unknown boot options get handed to init, unless they look like
 542 * unused parameters (modprobe will find them in /proc/cmdline).
 543 */
 544static int __init unknown_bootoption(char *param, char *val,
 545				     const char *unused, void *arg)
 546{
 547	size_t len = strlen(param);
 548	/*
 549	 * Well-known bootloader identifiers:
 550	 * 1. LILO/Grub pass "BOOT_IMAGE=...";
 551	 * 2. kexec/kdump (kexec-tools) pass "kexec".
 552	 */
 553	const char *bootloader[] = { "BOOT_IMAGE=", "kexec", NULL };
 554
 555	/* Handle params aliased to sysctls */
 556	if (sysctl_is_alias(param))
 557		return 0;
 558
 559	repair_env_string(param, val);
 560
 561	/* Handle bootloader identifier */
 562	for (int i = 0; bootloader[i]; i++) {
 563		if (strstarts(param, bootloader[i]))
 564			return 0;
 565	}
 566
 567	/* Handle obsolete-style parameters */
 568	if (obsolete_checksetup(param))
 569		return 0;
 570
 571	/* Unused module parameter. */
 572	if (strnchr(param, len, '.'))
 573		return 0;
 574
 575	if (panic_later)
 576		return 0;
 577
 578	if (val) {
 579		/* Environment option */
 580		unsigned int i;
 581		for (i = 0; envp_init[i]; i++) {
 582			if (i == MAX_INIT_ENVS) {
 583				panic_later = "env";
 584				panic_param = param;
 585			}
 586			if (!strncmp(param, envp_init[i], len+1))
 587				break;
 588		}
 589		envp_init[i] = param;
 590	} else {
 591		/* Command line option */
 592		unsigned int i;
 593		for (i = 0; argv_init[i]; i++) {
 594			if (i == MAX_INIT_ARGS) {
 595				panic_later = "init";
 596				panic_param = param;
 597			}
 598		}
 599		argv_init[i] = param;
 600	}
 601	return 0;
 602}
 603
 604static int __init init_setup(char *str)
 605{
 606	unsigned int i;
 607
 608	execute_command = str;
 609	/*
 610	 * In case LILO is going to boot us with default command line,
 611	 * it prepends "auto" before the whole cmdline which makes
 612	 * the shell think it should execute a script with such name.
 613	 * So we ignore all arguments entered _before_ init=... [MJ]
 614	 */
 615	for (i = 1; i < MAX_INIT_ARGS; i++)
 616		argv_init[i] = NULL;
 617	return 1;
 618}
 619__setup("init=", init_setup);
 620
 621static int __init rdinit_setup(char *str)
 622{
 623	unsigned int i;
 624
 625	ramdisk_execute_command = str;
 626	/* See "auto" comment in init_setup */
 627	for (i = 1; i < MAX_INIT_ARGS; i++)
 628		argv_init[i] = NULL;
 629	return 1;
 630}
 631__setup("rdinit=", rdinit_setup);
 632
 633#ifndef CONFIG_SMP
 634static inline void setup_nr_cpu_ids(void) { }
 635static inline void smp_prepare_cpus(unsigned int maxcpus) { }
 636#endif
 637
 638/*
 639 * We need to store the untouched command line for future reference.
 640 * We also need to store the touched command line since the parameter
 641 * parsing is performed in place, and we should allow a component to
 642 * store reference of name/value for future reference.
 643 */
 644static void __init setup_command_line(char *command_line)
 645{
 646	size_t len, xlen = 0, ilen = 0;
 647
 648	if (extra_command_line)
 649		xlen = strlen(extra_command_line);
 650	if (extra_init_args) {
 651		extra_init_args = strim(extra_init_args); /* remove trailing space */
 652		ilen = strlen(extra_init_args) + 4; /* for " -- " */
 653	}
 654
 655	len = xlen + strlen(boot_command_line) + ilen + 1;
 656
 657	saved_command_line = memblock_alloc_or_panic(len, SMP_CACHE_BYTES);
 658
 659	len = xlen + strlen(command_line) + 1;
 660
 661	static_command_line = memblock_alloc_or_panic(len, SMP_CACHE_BYTES);
 662
 663	if (xlen) {
 664		/*
 665		 * We have to put extra_command_line before boot command
 666		 * lines because there could be dashes (separator of init
 667		 * command line) in the command lines.
 668		 */
 669		strcpy(saved_command_line, extra_command_line);
 670		strcpy(static_command_line, extra_command_line);
 671	}
 672	strcpy(saved_command_line + xlen, boot_command_line);
 673	strcpy(static_command_line + xlen, command_line);
 674
 675	if (ilen) {
 676		/*
 677		 * Append supplemental init boot args to saved_command_line
 678		 * so that user can check what command line options passed
 679		 * to init.
 680		 * The order should always be
 681		 * " -- "[bootconfig init-param][cmdline init-param]
 682		 */
 683		if (initargs_offs) {
 684			len = xlen + initargs_offs;
 685			strcpy(saved_command_line + len, extra_init_args);
 686			len += ilen - 4;	/* strlen(extra_init_args) */
 687			strcpy(saved_command_line + len,
 688				boot_command_line + initargs_offs - 1);
 689		} else {
 690			len = strlen(saved_command_line);
 691			strcpy(saved_command_line + len, " -- ");
 692			len += 4;
 693			strcpy(saved_command_line + len, extra_init_args);
 694		}
 695	}
 696
 697	saved_command_line_len = strlen(saved_command_line);
 698}
 699
 700/*
 701 * We need to finalize in a non-__init function or else race conditions
 702 * between the root thread and the init thread may cause start_kernel to
 703 * be reaped by free_initmem before the root thread has proceeded to
 704 * cpu_idle.
 705 *
 706 * gcc-3.4 accidentally inlines this function, so use noinline.
 707 */
 708
 709static __initdata DECLARE_COMPLETION(kthreadd_done);
 710
 711static noinline void __ref __noreturn rest_init(void)
 712{
 713	struct task_struct *tsk;
 714	int pid;
 715
 716	rcu_scheduler_starting();
 717	/*
 718	 * We need to spawn init first so that it obtains pid 1, however
 719	 * the init task will end up wanting to create kthreads, which, if
 720	 * we schedule it before we create kthreadd, will OOPS.
 721	 */
 722	pid = user_mode_thread(kernel_init, NULL, CLONE_FS);
 723	/*
 724	 * Pin init on the boot CPU. Task migration is not properly working
 725	 * until sched_init_smp() has been run. It will set the allowed
 726	 * CPUs for init to the non isolated CPUs.
 727	 */
 728	rcu_read_lock();
 729	tsk = find_task_by_pid_ns(pid, &init_pid_ns);
 730	tsk->flags |= PF_NO_SETAFFINITY;
 731	set_cpus_allowed_ptr(tsk, cpumask_of(smp_processor_id()));
 732	rcu_read_unlock();
 733
 734	numa_default_policy();
 735	pid = kernel_thread(kthreadd, NULL, NULL, CLONE_FS | CLONE_FILES);
 736	rcu_read_lock();
 737	kthreadd_task = find_task_by_pid_ns(pid, &init_pid_ns);
 738	rcu_read_unlock();
 739
 740	/*
 741	 * Enable might_sleep() and smp_processor_id() checks.
 742	 * They cannot be enabled earlier because with CONFIG_PREEMPTION=y
 743	 * kernel_thread() would trigger might_sleep() splats. With
 744	 * CONFIG_PREEMPT_VOLUNTARY=y the init task might have scheduled
 745	 * already, but it's stuck on the kthreadd_done completion.
 746	 */
 747	system_state = SYSTEM_SCHEDULING;
 748
 749	complete(&kthreadd_done);
 750
 751	/*
 752	 * The boot idle thread must execute schedule()
 753	 * at least once to get things moving:
 754	 */
 755	schedule_preempt_disabled();
 756	/* Call into cpu_idle with preempt disabled */
 757	cpu_startup_entry(CPUHP_ONLINE);
 758}
 759
 760/* Check for early params. */
 761static int __init do_early_param(char *param, char *val,
 762				 const char *unused, void *arg)
 763{
 764	const struct obs_kernel_param *p;
 765
 766	for (p = __setup_start; p < __setup_end; p++) {
 767		if (p->early && parameq(param, p->str)) {
 768			if (p->setup_func(val) != 0)
 769				pr_warn("Malformed early option '%s'\n", param);
 770		}
 771	}
 772	/* We accept everything at this stage. */
 773	return 0;
 774}
 775
 776void __init parse_early_options(char *cmdline)
 777{
 778	parse_args("early options", cmdline, NULL, 0, 0, 0, NULL,
 779		   do_early_param);
 780}
 781
 782/* Arch code calls this early on, or if not, just before other parsing. */
 783void __init parse_early_param(void)
 784{
 785	static int done __initdata;
 786	static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;
 787
 788	if (done)
 789		return;
 790
 791	/* All fall through to do_early_param. */
 792	strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
 793	parse_early_options(tmp_cmdline);
 794	done = 1;
 795}
 796
 797void __init __weak arch_post_acpi_subsys_init(void) { }
 798
 799void __init __weak smp_setup_processor_id(void)
 800{
 801}
 802
 803void __init __weak smp_prepare_boot_cpu(void)
 804{
 805}
 806
 807# if THREAD_SIZE >= PAGE_SIZE
 808void __init __weak thread_stack_cache_init(void)
 809{
 810}
 811#endif
 812
 813void __init __weak poking_init(void) { }
 814
 815void __init __weak pgtable_cache_init(void) { }
 816
 817void __init __weak trap_init(void) { }
 818
 819bool initcall_debug;
 820core_param(initcall_debug, initcall_debug, bool, 0644);
 821
 822#ifdef TRACEPOINTS_ENABLED
 823static void __init initcall_debug_enable(void);
 824#else
 825static inline void initcall_debug_enable(void)
 826{
 827}
 828#endif
 829
 830#ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET
 831DEFINE_STATIC_KEY_MAYBE_RO(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT,
 832			   randomize_kstack_offset);
 833DEFINE_PER_CPU(u32, kstack_offset);
 834
 835static int __init early_randomize_kstack_offset(char *buf)
 836{
 837	int ret;
 838	bool bool_result;
 839
 840	ret = kstrtobool(buf, &bool_result);
 841	if (ret)
 842		return ret;
 843
 844	if (bool_result)
 845		static_branch_enable(&randomize_kstack_offset);
 846	else
 847		static_branch_disable(&randomize_kstack_offset);
 848	return 0;
 849}
 850early_param("randomize_kstack_offset", early_randomize_kstack_offset);
 851#endif
 852
 853static void __init print_unknown_bootoptions(void)
 854{
 855	char *unknown_options;
 856	char *end;
 857	const char *const *p;
 858	size_t len;
 859
 860	if (panic_later || (!argv_init[1] && !envp_init[2]))
 861		return;
 862
 863	/*
 864	 * Determine how many options we have to print out, plus a space
 865	 * before each
 866	 */
 867	len = 1; /* null terminator */
 868	for (p = &argv_init[1]; *p; p++) {
 869		len++;
 870		len += strlen(*p);
 871	}
 872	for (p = &envp_init[2]; *p; p++) {
 873		len++;
 874		len += strlen(*p);
 875	}
 876
 877	unknown_options = memblock_alloc(len, SMP_CACHE_BYTES);
 878	if (!unknown_options) {
 879		pr_err("%s: Failed to allocate %zu bytes\n",
 880			__func__, len);
 881		return;
 882	}
 883	end = unknown_options;
 884
 885	for (p = &argv_init[1]; *p; p++)
 886		end += sprintf(end, " %s", *p);
 887	for (p = &envp_init[2]; *p; p++)
 888		end += sprintf(end, " %s", *p);
 889
 890	/* Start at unknown_options[1] to skip the initial space */
 891	pr_notice("Unknown kernel command line parameters \"%s\", will be passed to user space.\n",
 892		&unknown_options[1]);
 893	memblock_free(unknown_options, len);
 894}
 895
 896static void __init early_numa_node_init(void)
 897{
 898#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
 899#ifndef cpu_to_node
 900	int cpu;
 901
 902	/* The early_cpu_to_node() should be ready here. */
 903	for_each_possible_cpu(cpu)
 904		set_cpu_numa_node(cpu, early_cpu_to_node(cpu));
 905#endif
 906#endif
 907}
 908
 909#define KERNEL_CMDLINE_PREFIX		"Kernel command line: "
 910#define KERNEL_CMDLINE_PREFIX_LEN	(sizeof(KERNEL_CMDLINE_PREFIX) - 1)
 911#define KERNEL_CMDLINE_CONTINUATION	" \\"
 912#define KERNEL_CMDLINE_CONTINUATION_LEN	(sizeof(KERNEL_CMDLINE_CONTINUATION) - 1)
 913
 914#define MIN_CMDLINE_LOG_WRAP_IDEAL_LEN	(KERNEL_CMDLINE_PREFIX_LEN + \
 915					 KERNEL_CMDLINE_CONTINUATION_LEN)
 916#define CMDLINE_LOG_WRAP_IDEAL_LEN	(CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN > \
 917					 MIN_CMDLINE_LOG_WRAP_IDEAL_LEN ? \
 918					 CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN : \
 919					 MIN_CMDLINE_LOG_WRAP_IDEAL_LEN)
 920
 921#define IDEAL_CMDLINE_LEN		(CMDLINE_LOG_WRAP_IDEAL_LEN - KERNEL_CMDLINE_PREFIX_LEN)
 922#define IDEAL_CMDLINE_SPLIT_LEN		(IDEAL_CMDLINE_LEN - KERNEL_CMDLINE_CONTINUATION_LEN)
 923
 924/**
 925 * print_kernel_cmdline() - Print the kernel cmdline with wrapping.
 926 * @cmdline: The cmdline to print.
 927 *
 928 * Print the kernel command line, trying to wrap based on the Kconfig knob
 929 * CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN.
 930 *
 931 * Wrapping is based on spaces, ignoring quotes. All lines are prefixed
 932 * with "Kernel command line: " and lines that are not the last line have
 933 * a " \" suffix added to them. The prefix and suffix count towards the
 934 * line length for wrapping purposes. The ideal length will be exceeded
 935 * if no appropriate place to wrap is found.
 936 *
 937 * Example output if CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN is 40:
 938 *   Kernel command line: loglevel=7 \
 939 *   Kernel command line: init=/sbin/init \
 940 *   Kernel command line: root=PARTUUID=8c3efc1a-768b-6642-8d0c-89eb782f19f0/PARTNROFF=1 \
 941 *   Kernel command line: rootwait ro \
 942 *   Kernel command line: my_quoted_arg="The \
 943 *   Kernel command line: quick brown fox \
 944 *   Kernel command line: jumps over the \
 945 *   Kernel command line: lazy dog."
 946 */
 947static void __init print_kernel_cmdline(const char *cmdline)
 948{
 949	size_t len;
 950
 951	/* Config option of 0 or anything longer than the max disables wrapping */
 952	if (CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN == 0 ||
 953	    IDEAL_CMDLINE_LEN >= COMMAND_LINE_SIZE - 1) {
 954		pr_notice("%s%s\n", KERNEL_CMDLINE_PREFIX, cmdline);
 955		return;
 956	}
 957
 958	len = strlen(cmdline);
 959	while (len > IDEAL_CMDLINE_LEN) {
 960		const char *first_space;
 961		const char *prev_cutoff;
 962		const char *cutoff;
 963		int to_print;
 964		size_t used;
 965
 966		/* Find the last ' ' that wouldn't make the line too long */
 967		prev_cutoff = NULL;
 968		cutoff = cmdline;
 969		while (true) {
 970			cutoff = strchr(cutoff + 1, ' ');
 971			if (!cutoff || cutoff - cmdline > IDEAL_CMDLINE_SPLIT_LEN)
 972				break;
 973			prev_cutoff = cutoff;
 974		}
 975		if (prev_cutoff)
 976			cutoff = prev_cutoff;
 977		else if (!cutoff)
 978			break;
 979
 980		/* Find the beginning and end of the string of spaces */
 981		first_space = cutoff;
 982		while (first_space > cmdline && first_space[-1] == ' ')
 983			first_space--;
 984		to_print = first_space - cmdline;
 985		while (*cutoff == ' ')
 986			cutoff++;
 987		used = cutoff - cmdline;
 988
 989		/* If the whole string is used, break and do the final printout */
 990		if (len == used)
 991			break;
 992
 993		if (to_print)
 994			pr_notice("%s%.*s%s\n", KERNEL_CMDLINE_PREFIX,
 995				  to_print, cmdline, KERNEL_CMDLINE_CONTINUATION);
 996
 997		len -= used;
 998		cmdline += used;
 999	}
1000	if (len)
1001		pr_notice("%s%s\n", KERNEL_CMDLINE_PREFIX, cmdline);
1002}
1003
1004asmlinkage __visible __init __no_sanitize_address __noreturn __no_stack_protector
1005void start_kernel(void)
1006{
1007	char *command_line;
1008	char *after_dashes;
1009
1010	set_task_stack_end_magic(&init_task);
1011	smp_setup_processor_id();
1012	debug_objects_early_init();
1013	init_vmlinux_build_id();
1014
1015	cgroup_init_early();
1016
1017	local_irq_disable();
1018	early_boot_irqs_disabled = true;
1019
1020	/*
1021	 * Interrupts are still disabled. Do necessary setups, then
1022	 * enable them.
1023	 */
1024	boot_cpu_init();
1025	page_address_init();
1026	pr_notice("%s", linux_banner);
1027	setup_arch(&command_line);
1028	/* Static keys and static calls are needed by LSMs */
1029	jump_label_init();
1030	static_call_init();
1031	early_security_init();
1032	setup_boot_config();
1033	setup_command_line(command_line);
1034	setup_nr_cpu_ids();
1035	setup_per_cpu_areas();
1036	smp_prepare_boot_cpu();	/* arch-specific boot-cpu hooks */
1037	early_numa_node_init();
1038	boot_cpu_hotplug_init();
1039
1040	print_kernel_cmdline(saved_command_line);
1041	/* parameters may set static keys */
1042	parse_early_param();
1043	after_dashes = parse_args("Booting kernel",
1044				  static_command_line, __start___param,
1045				  __stop___param - __start___param,
1046				  -1, -1, NULL, &unknown_bootoption);
1047	print_unknown_bootoptions();
1048	if (!IS_ERR_OR_NULL(after_dashes))
1049		parse_args("Setting init args", after_dashes, NULL, 0, -1, -1,
1050			   NULL, set_init_arg);
1051	if (extra_init_args)
1052		parse_args("Setting extra init args", extra_init_args,
1053			   NULL, 0, -1, -1, NULL, set_init_arg);
1054
1055	/* Architectural and non-timekeeping rng init, before allocator init */
1056	random_init_early(command_line);
1057
1058	/*
1059	 * These use large bootmem allocations and must precede
1060	 * initalization of page allocator
1061	 */
1062	setup_log_buf(0);
1063	vfs_caches_init_early();
1064	sort_main_extable();
1065	trap_init();
1066	mm_core_init();
1067	maple_tree_init();
1068	poking_init();
1069	ftrace_init();
1070
1071	/* trace_printk can be enabled here */
1072	early_trace_init();
1073
1074	/*
1075	 * Set up the scheduler prior starting any interrupts (such as the
1076	 * timer interrupt). Full topology setup happens at smp_init()
1077	 * time - but meanwhile we still have a functioning scheduler.
1078	 */
1079	sched_init();
1080
1081	if (WARN(!irqs_disabled(),
1082		 "Interrupts were enabled *very* early, fixing it\n"))
1083		local_irq_disable();
1084	radix_tree_init();
1085
1086	/*
1087	 * Set up housekeeping before setting up workqueues to allow the unbound
1088	 * workqueue to take non-housekeeping into account.
1089	 */
1090	housekeeping_init();
1091
1092	/*
1093	 * Allow workqueue creation and work item queueing/cancelling
1094	 * early.  Work item execution depends on kthreads and starts after
1095	 * workqueue_init().
1096	 */
1097	workqueue_init_early();
1098
1099	rcu_init();
1100	kvfree_rcu_init();
1101
1102	/* Trace events are available after this */
1103	trace_init();
1104
1105	if (initcall_debug)
1106		initcall_debug_enable();
1107
1108	context_tracking_init();
1109	/* init some links before init_ISA_irqs() */
1110	early_irq_init();
1111	init_IRQ();
1112	tick_init();
1113	rcu_init_nohz();
1114	timers_init();
1115	srcu_init();
1116	hrtimers_init();
1117	softirq_init();
1118	timekeeping_init();
1119	time_init();
1120
1121	/* This must be after timekeeping is initialized */
1122	random_init();
1123
1124	/* These make use of the fully initialized rng */
1125	kfence_init();
1126	boot_init_stack_canary();
1127
1128	perf_event_init();
1129	profile_init();
1130	call_function_init();
1131	WARN(!irqs_disabled(), "Interrupts were enabled early\n");
1132
1133	early_boot_irqs_disabled = false;
1134	local_irq_enable();
1135
1136	kmem_cache_init_late();
1137
1138	/*
1139	 * HACK ALERT! This is early. We're enabling the console before
1140	 * we've done PCI setups etc, and console_init() must be aware of
1141	 * this. But we do want output early, in case something goes wrong.
1142	 */
1143	console_init();
1144	if (panic_later)
1145		panic("Too many boot %s vars at `%s'", panic_later,
1146		      panic_param);
1147
1148	lockdep_init();
1149
1150	/*
1151	 * Need to run this when irqs are enabled, because it wants
1152	 * to self-test [hard/soft]-irqs on/off lock inversion bugs
1153	 * too:
1154	 */
1155	locking_selftest();
1156
1157#ifdef CONFIG_BLK_DEV_INITRD
1158	if (initrd_start && !initrd_below_start_ok &&
1159	    page_to_pfn(virt_to_page((void *)initrd_start)) < min_low_pfn) {
1160		pr_crit("initrd overwritten (0x%08lx < 0x%08lx) - disabling it.\n",
1161		    page_to_pfn(virt_to_page((void *)initrd_start)),
1162		    min_low_pfn);
1163		initrd_start = 0;
1164	}
1165#endif
1166	setup_per_cpu_pageset();
1167	numa_policy_init();
1168	acpi_early_init();
1169	if (late_time_init)
1170		late_time_init();
1171	sched_clock_init();
1172	calibrate_delay();
1173
1174	arch_cpu_finalize_init();
1175
1176	pid_idr_init();
1177	anon_vma_init();
1178	thread_stack_cache_init();
1179	cred_init();
1180	fork_init();
1181	proc_caches_init();
1182	uts_ns_init();
1183	time_ns_init();
1184	key_init();
1185	security_init();
1186	dbg_late_init();
1187	net_ns_init();
1188	vfs_caches_init();
1189	pagecache_init();
1190	signals_init();
1191	seq_file_init();
1192	proc_root_init();
1193	nsfs_init();
1194	pidfs_init();
1195	cpuset_init();
1196	mem_cgroup_init();
1197	cgroup_init();
1198	taskstats_init_early();
1199	delayacct_init();
1200
1201	acpi_subsystem_init();
1202	arch_post_acpi_subsys_init();
1203	kcsan_init();
1204
1205	/* Do the rest non-__init'ed, we're now alive */
1206	rest_init();
1207
1208	/*
1209	 * Avoid stack canaries in callers of boot_init_stack_canary for gcc-10
1210	 * and older.
1211	 */
1212#if !__has_attribute(__no_stack_protector__)
1213	prevent_tail_call_optimization();
1214#endif
1215}
1216
1217/* Call all constructor functions linked into the kernel. */
1218static void __init do_ctors(void)
1219{
1220/*
1221 * For UML, the constructors have already been called by the
1222 * normal setup code as it's just a normal ELF binary, so we
1223 * cannot do it again - but we do need CONFIG_CONSTRUCTORS
1224 * even on UML for modules.
1225 */
1226#if defined(CONFIG_CONSTRUCTORS) && !defined(CONFIG_UML)
1227	ctor_fn_t *fn = (ctor_fn_t *) __ctors_start;
1228
1229	for (; fn < (ctor_fn_t *) __ctors_end; fn++)
1230		(*fn)();
1231#endif
1232}
1233
1234#ifdef CONFIG_KALLSYMS
1235struct blacklist_entry {
1236	struct list_head next;
1237	char *buf;
1238};
1239
1240static __initdata_or_module LIST_HEAD(blacklisted_initcalls);
1241
1242static int __init initcall_blacklist(char *str)
1243{
1244	char *str_entry;
1245	struct blacklist_entry *entry;
1246
1247	/* str argument is a comma-separated list of functions */
1248	do {
1249		str_entry = strsep(&str, ",");
1250		if (str_entry) {
1251			pr_debug("blacklisting initcall %s\n", str_entry);
1252			entry = memblock_alloc_or_panic(sizeof(*entry),
1253					       SMP_CACHE_BYTES);
1254			entry->buf = memblock_alloc_or_panic(strlen(str_entry) + 1,
1255						    SMP_CACHE_BYTES);
1256			strcpy(entry->buf, str_entry);
1257			list_add(&entry->next, &blacklisted_initcalls);
1258		}
1259	} while (str_entry);
1260
1261	return 1;
1262}
1263
1264static bool __init_or_module initcall_blacklisted(initcall_t fn)
1265{
1266	struct blacklist_entry *entry;
1267	char fn_name[KSYM_SYMBOL_LEN];
1268	unsigned long addr;
1269
1270	if (list_empty(&blacklisted_initcalls))
1271		return false;
1272
1273	addr = (unsigned long) dereference_function_descriptor(fn);
1274	sprint_symbol_no_offset(fn_name, addr);
1275
1276	/*
1277	 * fn will be "function_name [module_name]" where [module_name] is not
1278	 * displayed for built-in init functions.  Strip off the [module_name].
1279	 */
1280	strreplace(fn_name, ' ', '\0');
1281
1282	list_for_each_entry(entry, &blacklisted_initcalls, next) {
1283		if (!strcmp(fn_name, entry->buf)) {
1284			pr_debug("initcall %s blacklisted\n", fn_name);
1285			return true;
1286		}
1287	}
1288
1289	return false;
1290}
1291#else
1292static int __init initcall_blacklist(char *str)
1293{
1294	pr_warn("initcall_blacklist requires CONFIG_KALLSYMS\n");
1295	return 0;
1296}
1297
1298static bool __init_or_module initcall_blacklisted(initcall_t fn)
1299{
1300	return false;
1301}
1302#endif
1303__setup("initcall_blacklist=", initcall_blacklist);
1304
1305static __init_or_module void
1306trace_initcall_start_cb(void *data, initcall_t fn)
1307{
1308	ktime_t *calltime = data;
1309
1310	printk(KERN_DEBUG "calling  %pS @ %i\n", fn, task_pid_nr(current));
1311	*calltime = ktime_get();
1312}
1313
1314static __init_or_module void
1315trace_initcall_finish_cb(void *data, initcall_t fn, int ret)
1316{
1317	ktime_t rettime, *calltime = data;
1318
1319	rettime = ktime_get();
1320	printk(KERN_DEBUG "initcall %pS returned %d after %lld usecs\n",
1321		 fn, ret, (unsigned long long)ktime_us_delta(rettime, *calltime));
1322}
1323
1324static __init_or_module void
1325trace_initcall_level_cb(void *data, const char *level)
1326{
1327	printk(KERN_DEBUG "entering initcall level: %s\n", level);
1328}
1329
1330static ktime_t initcall_calltime;
1331
1332#ifdef TRACEPOINTS_ENABLED
1333static void __init initcall_debug_enable(void)
1334{
1335	int ret;
1336
1337	ret = register_trace_initcall_start(trace_initcall_start_cb,
1338					    &initcall_calltime);
1339	ret |= register_trace_initcall_finish(trace_initcall_finish_cb,
1340					      &initcall_calltime);
1341	ret |= register_trace_initcall_level(trace_initcall_level_cb, NULL);
1342	WARN(ret, "Failed to register initcall tracepoints\n");
1343}
1344# define do_trace_initcall_start	trace_initcall_start
1345# define do_trace_initcall_finish	trace_initcall_finish
1346# define do_trace_initcall_level	trace_initcall_level
1347#else
1348static inline void do_trace_initcall_start(initcall_t fn)
1349{
1350	if (!initcall_debug)
1351		return;
1352	trace_initcall_start_cb(&initcall_calltime, fn);
1353}
1354static inline void do_trace_initcall_finish(initcall_t fn, int ret)
1355{
1356	if (!initcall_debug)
1357		return;
1358	trace_initcall_finish_cb(&initcall_calltime, fn, ret);
1359}
1360static inline void do_trace_initcall_level(const char *level)
1361{
1362	if (!initcall_debug)
1363		return;
1364	trace_initcall_level_cb(NULL, level);
1365}
1366#endif /* !TRACEPOINTS_ENABLED */
1367
1368int __init_or_module do_one_initcall(initcall_t fn)
1369{
1370	int count = preempt_count();
1371	char msgbuf[64];
1372	int ret;
1373
1374	if (initcall_blacklisted(fn))
1375		return -EPERM;
1376
1377	do_trace_initcall_start(fn);
1378	ret = fn();
1379	do_trace_initcall_finish(fn, ret);
1380
1381	msgbuf[0] = 0;
1382
1383	if (preempt_count() != count) {
1384		sprintf(msgbuf, "preemption imbalance ");
1385		preempt_count_set(count);
1386	}
1387	if (irqs_disabled()) {
1388		strlcat(msgbuf, "disabled interrupts ", sizeof(msgbuf));
1389		local_irq_enable();
1390	}
1391	WARN(msgbuf[0], "initcall %pS returned with %s\n", fn, msgbuf);
1392
1393	add_latent_entropy();
1394	return ret;
1395}
1396
1397
1398static initcall_entry_t *initcall_levels[] __initdata = {
1399	__initcall0_start,
1400	__initcall1_start,
1401	__initcall2_start,
1402	__initcall3_start,
1403	__initcall4_start,
1404	__initcall5_start,
1405	__initcall6_start,
1406	__initcall7_start,
1407	__initcall_end,
1408};
1409
1410/* Keep these in sync with initcalls in include/linux/init.h */
1411static const char *initcall_level_names[] __initdata = {
1412	"pure",
1413	"core",
1414	"postcore",
1415	"arch",
1416	"subsys",
1417	"fs",
1418	"device",
1419	"late",
1420};
1421
1422static int __init ignore_unknown_bootoption(char *param, char *val,
1423			       const char *unused, void *arg)
1424{
1425	return 0;
1426}
1427
1428static void __init do_initcall_level(int level, char *command_line)
1429{
1430	initcall_entry_t *fn;
1431
1432	parse_args(initcall_level_names[level],
1433		   command_line, __start___param,
1434		   __stop___param - __start___param,
1435		   level, level,
1436		   NULL, ignore_unknown_bootoption);
1437
1438	do_trace_initcall_level(initcall_level_names[level]);
1439	for (fn = initcall_levels[level]; fn < initcall_levels[level+1]; fn++)
1440		do_one_initcall(initcall_from_entry(fn));
1441}
1442
1443static void __init do_initcalls(void)
1444{
1445	int level;
1446	size_t len = saved_command_line_len + 1;
1447	char *command_line;
1448
1449	command_line = kzalloc(len, GFP_KERNEL);
1450	if (!command_line)
1451		panic("%s: Failed to allocate %zu bytes\n", __func__, len);
1452
1453	for (level = 0; level < ARRAY_SIZE(initcall_levels) - 1; level++) {
1454		/* Parser modifies command_line, restore it each time */
1455		strcpy(command_line, saved_command_line);
1456		do_initcall_level(level, command_line);
1457	}
1458
1459	kfree(command_line);
1460}
1461
1462/*
1463 * Ok, the machine is now initialized. None of the devices
1464 * have been touched yet, but the CPU subsystem is up and
1465 * running, and memory and process management works.
1466 *
1467 * Now we can finally start doing some real work..
1468 */
1469static void __init do_basic_setup(void)
1470{
1471	cpuset_init_smp();
1472	driver_init();
1473	init_irq_proc();
1474	do_ctors();
1475	do_initcalls();
1476}
1477
1478static void __init do_pre_smp_initcalls(void)
1479{
1480	initcall_entry_t *fn;
1481
1482	do_trace_initcall_level("early");
1483	for (fn = __initcall_start; fn < __initcall0_start; fn++)
1484		do_one_initcall(initcall_from_entry(fn));
1485}
1486
1487static int run_init_process(const char *init_filename)
1488{
1489	const char *const *p;
1490
1491	argv_init[0] = init_filename;
1492	pr_info("Run %s as init process\n", init_filename);
1493	pr_debug("  with arguments:\n");
1494	for (p = argv_init; *p; p++)
1495		pr_debug("    %s\n", *p);
1496	pr_debug("  with environment:\n");
1497	for (p = envp_init; *p; p++)
1498		pr_debug("    %s\n", *p);
1499	return kernel_execve(init_filename, argv_init, envp_init);
1500}
1501
1502static int try_to_run_init_process(const char *init_filename)
1503{
1504	int ret;
1505
1506	ret = run_init_process(init_filename);
1507
1508	if (ret && ret != -ENOENT) {
1509		pr_err("Starting init: %s exists but couldn't execute it (error %d)\n",
1510		       init_filename, ret);
1511	}
1512
1513	return ret;
1514}
1515
1516static noinline void __init kernel_init_freeable(void);
1517
1518#if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_STRICT_MODULE_RWX)
1519bool rodata_enabled __ro_after_init = true;
1520
1521#ifndef arch_parse_debug_rodata
1522static inline bool arch_parse_debug_rodata(char *str) { return false; }
1523#endif
1524
1525static int __init set_debug_rodata(char *str)
1526{
1527	if (arch_parse_debug_rodata(str))
1528		return 0;
1529
1530	if (str && !strcmp(str, "on"))
1531		rodata_enabled = true;
1532	else if (str && !strcmp(str, "off"))
1533		rodata_enabled = false;
1534	else
1535		pr_warn("Invalid option string for rodata: '%s'\n", str);
1536	return 0;
1537}
1538early_param("rodata", set_debug_rodata);
1539#endif
1540
1541static void mark_readonly(void)
1542{
1543	if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX) && rodata_enabled) {
1544		/*
1545		 * load_module() results in W+X mappings, which are cleaned
1546		 * up with init_free_wq. Let's make sure that queued work is
1547		 * flushed so that we don't hit false positives looking for
1548		 * insecure pages which are W+X.
1549		 */
1550		flush_module_init_free_work();
1551		jump_label_init_ro();
1552		mark_rodata_ro();
1553		debug_checkwx();
1554		rodata_test();
1555	} else if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) {
1556		pr_info("Kernel memory protection disabled.\n");
1557	} else if (IS_ENABLED(CONFIG_ARCH_HAS_STRICT_KERNEL_RWX)) {
1558		pr_warn("Kernel memory protection not selected by kernel config.\n");
1559	} else {
1560		pr_warn("This architecture does not have kernel memory protection.\n");
1561	}
1562}
1563
1564void __weak free_initmem(void)
1565{
1566	free_initmem_default(POISON_FREE_INITMEM);
1567}
1568
1569static int __ref kernel_init(void *unused)
1570{
1571	int ret;
1572
1573	/*
1574	 * Wait until kthreadd is all set-up.
1575	 */
1576	wait_for_completion(&kthreadd_done);
1577
1578	kernel_init_freeable();
1579	/* need to finish all async __init code before freeing the memory */
1580	async_synchronize_full();
1581
1582	system_state = SYSTEM_FREEING_INITMEM;
1583	kprobe_free_init_mem();
1584	ftrace_free_init_mem();
1585	kgdb_free_init_mem();
1586	exit_boot_config();
1587	free_initmem();
1588	mark_readonly();
1589
1590	/*
1591	 * Kernel mappings are now finalized - update the userspace page-table
1592	 * to finalize PTI.
1593	 */
1594	pti_finalize();
1595
1596	system_state = SYSTEM_RUNNING;
1597	numa_default_policy();
1598
1599	rcu_end_inkernel_boot();
1600
1601	do_sysctl_args();
1602
1603	if (ramdisk_execute_command) {
1604		ret = run_init_process(ramdisk_execute_command);
1605		if (!ret)
1606			return 0;
1607		pr_err("Failed to execute %s (error %d)\n",
1608		       ramdisk_execute_command, ret);
1609	}
1610
1611	/*
1612	 * We try each of these until one succeeds.
1613	 *
1614	 * The Bourne shell can be used instead of init if we are
1615	 * trying to recover a really broken machine.
1616	 */
1617	if (execute_command) {
1618		ret = run_init_process(execute_command);
1619		if (!ret)
1620			return 0;
1621		panic("Requested init %s failed (error %d).",
1622		      execute_command, ret);
1623	}
1624
1625	if (CONFIG_DEFAULT_INIT[0] != '\0') {
1626		ret = run_init_process(CONFIG_DEFAULT_INIT);
1627		if (ret)
1628			pr_err("Default init %s failed (error %d)\n",
1629			       CONFIG_DEFAULT_INIT, ret);
1630		else
1631			return 0;
1632	}
1633
1634	if (!try_to_run_init_process("/sbin/init") ||
1635	    !try_to_run_init_process("/etc/init") ||
1636	    !try_to_run_init_process("/bin/init") ||
1637	    !try_to_run_init_process("/bin/sh"))
1638		return 0;
1639
1640	panic("No working init found.  Try passing init= option to kernel. "
1641	      "See Linux Documentation/admin-guide/init.rst for guidance.");
1642}
1643
1644/* Open /dev/console, for stdin/stdout/stderr, this should never fail */
1645void __init console_on_rootfs(void)
1646{
1647	struct file *file = filp_open("/dev/console", O_RDWR, 0);
1648
1649	if (IS_ERR(file)) {
1650		pr_err("Warning: unable to open an initial console.\n");
1651		return;
1652	}
1653	init_dup(file);
1654	init_dup(file);
1655	init_dup(file);
1656	fput(file);
1657}
1658
1659static noinline void __init kernel_init_freeable(void)
1660{
1661	/* Now the scheduler is fully set up and can do blocking allocations */
1662	gfp_allowed_mask = __GFP_BITS_MASK;
1663
1664	/*
1665	 * init can allocate pages on any node
1666	 */
1667	set_mems_allowed(node_states[N_MEMORY]);
1668
1669	cad_pid = get_pid(task_pid(current));
1670
1671	smp_prepare_cpus(setup_max_cpus);
1672
1673	workqueue_init();
1674
1675	init_mm_internals();
1676
1677	do_pre_smp_initcalls();
1678	lockup_detector_init();
1679
1680	smp_init();
1681	sched_init_smp();
1682
1683	workqueue_init_topology();
1684	async_init();
1685	padata_init();
1686	page_alloc_init_late();
1687
1688	do_basic_setup();
1689
1690	kunit_run_all_tests();
1691
1692	wait_for_initramfs();
1693	console_on_rootfs();
1694
1695	/*
1696	 * check if there is an early userspace init.  If yes, let it do all
1697	 * the work
1698	 */
1699	int ramdisk_command_access;
1700	ramdisk_command_access = init_eaccess(ramdisk_execute_command);
1701	if (ramdisk_command_access != 0) {
1702		pr_warn("check access for rdinit=%s failed: %i, ignoring\n",
1703			ramdisk_execute_command, ramdisk_command_access);
1704		ramdisk_execute_command = NULL;
1705		prepare_namespace();
1706	}
1707
1708	/*
1709	 * Ok, we have completed the initial bootup, and
1710	 * we're essentially up and running. Get rid of the
1711	 * initmem segments and start the user-mode stuff..
1712	 *
1713	 * rootfs is available now, try loading the public keys
1714	 * and default modules
1715	 */
1716
1717	integrity_load_keys();
1718}