verifier.c

    1// SPDX-License-Identifier: GPL-2.0-only
    2/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
    3 * Copyright (c) 2016 Facebook
    4 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
    5 */
    6#include <uapi/linux/btf.h>
    7#include <linux/bpf-cgroup.h>
    8#include <linux/kernel.h>
    9#include <linux/types.h>
   10#include <linux/slab.h>
   11#include <linux/bpf.h>
   12#include <linux/btf.h>
   13#include <linux/bpf_verifier.h>
   14#include <linux/filter.h>
   15#include <net/netlink.h>
   16#include <linux/file.h>
   17#include <linux/vmalloc.h>
   18#include <linux/stringify.h>
   19#include <linux/bsearch.h>
   20#include <linux/sort.h>
   21#include <linux/perf_event.h>
   22#include <linux/ctype.h>
   23#include <linux/error-injection.h>
   24#include <linux/bpf_lsm.h>
   25#include <linux/btf_ids.h>
   26#include <linux/poison.h>
   27#include <linux/module.h>
   28#include <linux/cpumask.h>
   29#include <linux/bpf_mem_alloc.h>
   30#include <net/xdp.h>
   31#include <linux/trace_events.h>
   32#include <linux/kallsyms.h>
   33
   34#include "disasm.h"
   35
   36static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
   37#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
   38	[_id] = & _name ## _verifier_ops,
   39#define BPF_MAP_TYPE(_id, _ops)
   40#define BPF_LINK_TYPE(_id, _name)
   41#include <linux/bpf_types.h>
   42#undef BPF_PROG_TYPE
   43#undef BPF_MAP_TYPE
   44#undef BPF_LINK_TYPE
   45};
   46
   47enum bpf_features {
   48	BPF_FEAT_RDONLY_CAST_TO_VOID = 0,
   49	BPF_FEAT_STREAMS	     = 1,
   50	__MAX_BPF_FEAT,
   51};
   52
   53struct bpf_mem_alloc bpf_global_percpu_ma;
   54static bool bpf_global_percpu_ma_set;
   55
   56/* bpf_check() is a static code analyzer that walks eBPF program
   57 * instruction by instruction and updates register/stack state.
   58 * All paths of conditional branches are analyzed until 'bpf_exit' insn.
   59 *
   60 * The first pass is depth-first-search to check that the program is a DAG.
   61 * It rejects the following programs:
   62 * - larger than BPF_MAXINSNS insns
   63 * - if loop is present (detected via back-edge)
   64 * - unreachable insns exist (shouldn't be a forest. program = one function)
   65 * - out of bounds or malformed jumps
   66 * The second pass is all possible path descent from the 1st insn.
   67 * Since it's analyzing all paths through the program, the length of the
   68 * analysis is limited to 64k insn, which may be hit even if total number of
   69 * insn is less then 4K, but there are too many branches that change stack/regs.
   70 * Number of 'branches to be analyzed' is limited to 1k
   71 *
   72 * On entry to each instruction, each register has a type, and the instruction
   73 * changes the types of the registers depending on instruction semantics.
   74 * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is
   75 * copied to R1.
   76 *
   77 * All registers are 64-bit.
   78 * R0 - return register
   79 * R1-R5 argument passing registers
   80 * R6-R9 callee saved registers
   81 * R10 - frame pointer read-only
   82 *
   83 * At the start of BPF program the register R1 contains a pointer to bpf_context
   84 * and has type PTR_TO_CTX.
   85 *
   86 * Verifier tracks arithmetic operations on pointers in case:
   87 *    BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
   88 *    BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20),
   89 * 1st insn copies R10 (which has FRAME_PTR) type into R1
   90 * and 2nd arithmetic instruction is pattern matched to recognize
   91 * that it wants to construct a pointer to some element within stack.
   92 * So after 2nd insn, the register R1 has type PTR_TO_STACK
   93 * (and -20 constant is saved for further stack bounds checking).
   94 * Meaning that this reg is a pointer to stack plus known immediate constant.
   95 *
   96 * Most of the time the registers have SCALAR_VALUE type, which
   97 * means the register has some value, but it's not a valid pointer.
   98 * (like pointer plus pointer becomes SCALAR_VALUE type)
   99 *
  100 * When verifier sees load or store instructions the type of base register
  101 * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are
  102 * four pointer types recognized by check_mem_access() function.
  103 *
  104 * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value'
  105 * and the range of [ptr, ptr + map's value_size) is accessible.
  106 *
  107 * registers used to pass values to function calls are checked against
  108 * function argument constraints.
  109 *
  110 * ARG_PTR_TO_MAP_KEY is one of such argument constraints.
  111 * It means that the register type passed to this function must be
  112 * PTR_TO_STACK and it will be used inside the function as
  113 * 'pointer to map element key'
  114 *
  115 * For example the argument constraints for bpf_map_lookup_elem():
  116 *   .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
  117 *   .arg1_type = ARG_CONST_MAP_PTR,
  118 *   .arg2_type = ARG_PTR_TO_MAP_KEY,
  119 *
  120 * ret_type says that this function returns 'pointer to map elem value or null'
  121 * function expects 1st argument to be a const pointer to 'struct bpf_map' and
  122 * 2nd argument should be a pointer to stack, which will be used inside
  123 * the helper function as a pointer to map element key.
  124 *
  125 * On the kernel side the helper function looks like:
  126 * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
  127 * {
  128 *    struct bpf_map *map = (struct bpf_map *) (unsigned long) r1;
  129 *    void *key = (void *) (unsigned long) r2;
  130 *    void *value;
  131 *
  132 *    here kernel can access 'key' and 'map' pointers safely, knowing that
  133 *    [key, key + map->key_size) bytes are valid and were initialized on
  134 *    the stack of eBPF program.
  135 * }
  136 *
  137 * Corresponding eBPF program may look like:
  138 *    BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),  // after this insn R2 type is FRAME_PTR
  139 *    BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK
  140 *    BPF_LD_MAP_FD(BPF_REG_1, map_fd),      // after this insn R1 type is CONST_PTR_TO_MAP
  141 *    BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
  142 * here verifier looks at prototype of map_lookup_elem() and sees:
  143 * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok,
  144 * Now verifier knows that this map has key of R1->map_ptr->key_size bytes
  145 *
  146 * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far,
  147 * Now verifier checks that [R2, R2 + map's key_size) are within stack limits
  148 * and were initialized prior to this call.
  149 * If it's ok, then verifier allows this BPF_CALL insn and looks at
  150 * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets
  151 * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function
  152 * returns either pointer to map value or NULL.
  153 *
  154 * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off'
  155 * insn, the register holding that pointer in the true branch changes state to
  156 * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false
  157 * branch. See check_cond_jmp_op().
  158 *
  159 * After the call R0 is set to return type of the function and registers R1-R5
  160 * are set to NOT_INIT to indicate that they are no longer readable.
  161 *
  162 * The following reference types represent a potential reference to a kernel
  163 * resource which, after first being allocated, must be checked and freed by
  164 * the BPF program:
  165 * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET
  166 *
  167 * When the verifier sees a helper call return a reference type, it allocates a
  168 * pointer id for the reference and stores it in the current function state.
  169 * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into
  170 * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type
  171 * passes through a NULL-check conditional. For the branch wherein the state is
  172 * changed to CONST_IMM, the verifier releases the reference.
  173 *
  174 * For each helper function that allocates a reference, such as
  175 * bpf_sk_lookup_tcp(), there is a corresponding release function, such as
  176 * bpf_sk_release(). When a reference type passes into the release function,
  177 * the verifier also releases the reference. If any unchecked or unreleased
  178 * reference remains at the end of the program, the verifier rejects it.
  179 */
  180
  181/* verifier_state + insn_idx are pushed to stack when branch is encountered */
  182struct bpf_verifier_stack_elem {
  183	/* verifier state is 'st'
  184	 * before processing instruction 'insn_idx'
  185	 * and after processing instruction 'prev_insn_idx'
  186	 */
  187	struct bpf_verifier_state st;
  188	int insn_idx;
  189	int prev_insn_idx;
  190	struct bpf_verifier_stack_elem *next;
  191	/* length of verifier log at the time this state was pushed on stack */
  192	u32 log_pos;
  193};
  194
  195#define BPF_COMPLEXITY_LIMIT_JMP_SEQ	8192
  196#define BPF_COMPLEXITY_LIMIT_STATES	64
  197
  198#define BPF_MAP_KEY_POISON	(1ULL << 63)
  199#define BPF_MAP_KEY_SEEN	(1ULL << 62)
  200
  201#define BPF_GLOBAL_PERCPU_MA_MAX_SIZE  512
  202
  203#define BPF_PRIV_STACK_MIN_SIZE		64
  204
  205static int acquire_reference(struct bpf_verifier_env *env, int insn_idx);
  206static int release_reference_nomark(struct bpf_verifier_state *state, int ref_obj_id);
  207static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);
  208static void invalidate_non_owning_refs(struct bpf_verifier_env *env);
  209static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env);
  210static int ref_set_non_owning(struct bpf_verifier_env *env,
  211			      struct bpf_reg_state *reg);
  212static bool is_trusted_reg(const struct bpf_reg_state *reg);
  213
  214static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
  215{
  216	return aux->map_ptr_state.poison;
  217}
  218
  219static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)
  220{
  221	return aux->map_ptr_state.unpriv;
  222}
  223
  224static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux,
  225			      struct bpf_map *map,
  226			      bool unpriv, bool poison)
  227{
  228	unpriv |= bpf_map_ptr_unpriv(aux);
  229	aux->map_ptr_state.unpriv = unpriv;
  230	aux->map_ptr_state.poison = poison;
  231	aux->map_ptr_state.map_ptr = map;
  232}
  233
  234static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux)
  235{
  236	return aux->map_key_state & BPF_MAP_KEY_POISON;
  237}
  238
  239static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux)
  240{
  241	return !(aux->map_key_state & BPF_MAP_KEY_SEEN);
  242}
  243
  244static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux)
  245{
  246	return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON);
  247}
  248
  249static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state)
  250{
  251	bool poisoned = bpf_map_key_poisoned(aux);
  252
  253	aux->map_key_state = state | BPF_MAP_KEY_SEEN |
  254			     (poisoned ? BPF_MAP_KEY_POISON : 0ULL);
  255}
  256
  257static bool bpf_helper_call(const struct bpf_insn *insn)
  258{
  259	return insn->code == (BPF_JMP | BPF_CALL) &&
  260	       insn->src_reg == 0;
  261}
  262
  263static bool bpf_pseudo_call(const struct bpf_insn *insn)
  264{
  265	return insn->code == (BPF_JMP | BPF_CALL) &&
  266	       insn->src_reg == BPF_PSEUDO_CALL;
  267}
  268
  269static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
  270{
  271	return insn->code == (BPF_JMP | BPF_CALL) &&
  272	       insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
  273}
  274
  275struct bpf_call_arg_meta {
  276	struct bpf_map *map_ptr;
  277	bool raw_mode;
  278	bool pkt_access;
  279	u8 release_regno;
  280	int regno;
  281	int access_size;
  282	int mem_size;
  283	u64 msize_max_value;
  284	int ref_obj_id;
  285	int dynptr_id;
  286	int map_uid;
  287	int func_id;
  288	struct btf *btf;
  289	u32 btf_id;
  290	struct btf *ret_btf;
  291	u32 ret_btf_id;
  292	u32 subprogno;
  293	struct btf_field *kptr_field;
  294	s64 const_map_key;
  295};
  296
  297struct bpf_kfunc_call_arg_meta {
  298	/* In parameters */
  299	struct btf *btf;
  300	u32 func_id;
  301	u32 kfunc_flags;
  302	const struct btf_type *func_proto;
  303	const char *func_name;
  304	/* Out parameters */
  305	u32 ref_obj_id;
  306	u8 release_regno;
  307	bool r0_rdonly;
  308	u32 ret_btf_id;
  309	u64 r0_size;
  310	u32 subprogno;
  311	struct {
  312		u64 value;
  313		bool found;
  314	} arg_constant;
  315
  316	/* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling,
  317	 * generally to pass info about user-defined local kptr types to later
  318	 * verification logic
  319	 *   bpf_obj_drop/bpf_percpu_obj_drop
  320	 *     Record the local kptr type to be drop'd
  321	 *   bpf_refcount_acquire (via KF_ARG_PTR_TO_REFCOUNTED_KPTR arg type)
  322	 *     Record the local kptr type to be refcount_incr'd and use
  323	 *     arg_owning_ref to determine whether refcount_acquire should be
  324	 *     fallible
  325	 */
  326	struct btf *arg_btf;
  327	u32 arg_btf_id;
  328	bool arg_owning_ref;
  329	bool arg_prog;
  330
  331	struct {
  332		struct btf_field *field;
  333	} arg_list_head;
  334	struct {
  335		struct btf_field *field;
  336	} arg_rbtree_root;
  337	struct {
  338		enum bpf_dynptr_type type;
  339		u32 id;
  340		u32 ref_obj_id;
  341	} initialized_dynptr;
  342	struct {
  343		u8 spi;
  344		u8 frameno;
  345	} iter;
  346	struct {
  347		struct bpf_map *ptr;
  348		int uid;
  349	} map;
  350	u64 mem_size;
  351};
  352
  353struct btf *btf_vmlinux;
  354
  355static const char *btf_type_name(const struct btf *btf, u32 id)
  356{
  357	return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
  358}
  359
  360static DEFINE_MUTEX(bpf_verifier_lock);
  361static DEFINE_MUTEX(bpf_percpu_ma_lock);
  362
  363__printf(2, 3) static void verbose(void *private_data, const char *fmt, ...)
  364{
  365	struct bpf_verifier_env *env = private_data;
  366	va_list args;
  367
  368	if (!bpf_verifier_log_needed(&env->log))
  369		return;
  370
  371	va_start(args, fmt);
  372	bpf_verifier_vlog(&env->log, fmt, args);
  373	va_end(args);
  374}
  375
  376static void verbose_invalid_scalar(struct bpf_verifier_env *env,
  377				   struct bpf_reg_state *reg,
  378				   struct bpf_retval_range range, const char *ctx,
  379				   const char *reg_name)
  380{
  381	bool unknown = true;
  382
  383	verbose(env, "%s the register %s has", ctx, reg_name);
  384	if (reg->smin_value > S64_MIN) {
  385		verbose(env, " smin=%lld", reg->smin_value);
  386		unknown = false;
  387	}
  388	if (reg->smax_value < S64_MAX) {
  389		verbose(env, " smax=%lld", reg->smax_value);
  390		unknown = false;
  391	}
  392	if (unknown)
  393		verbose(env, " unknown scalar value");
  394	verbose(env, " should have been in [%d, %d]\n", range.minval, range.maxval);
  395}
  396
  397static bool reg_not_null(const struct bpf_reg_state *reg)
  398{
  399	enum bpf_reg_type type;
  400
  401	type = reg->type;
  402	if (type_may_be_null(type))
  403		return false;
  404
  405	type = base_type(type);
  406	return type == PTR_TO_SOCKET ||
  407		type == PTR_TO_TCP_SOCK ||
  408		type == PTR_TO_MAP_VALUE ||
  409		type == PTR_TO_MAP_KEY ||
  410		type == PTR_TO_SOCK_COMMON ||
  411		(type == PTR_TO_BTF_ID && is_trusted_reg(reg)) ||
  412		(type == PTR_TO_MEM && !(reg->type & PTR_UNTRUSTED)) ||
  413		type == CONST_PTR_TO_MAP;
  414}
  415
  416static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg)
  417{
  418	struct btf_record *rec = NULL;
  419	struct btf_struct_meta *meta;
  420
  421	if (reg->type == PTR_TO_MAP_VALUE) {
  422		rec = reg->map_ptr->record;
  423	} else if (type_is_ptr_alloc_obj(reg->type)) {
  424		meta = btf_find_struct_meta(reg->btf, reg->btf_id);
  425		if (meta)
  426			rec = meta->record;
  427	}
  428	return rec;
  429}
  430
  431static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog)
  432{
  433	struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux;
  434
  435	return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL;
  436}
  437
  438static const char *subprog_name(const struct bpf_verifier_env *env, int subprog)
  439{
  440	struct bpf_func_info *info;
  441
  442	if (!env->prog->aux->func_info)
  443		return "";
  444
  445	info = &env->prog->aux->func_info[subprog];
  446	return btf_type_name(env->prog->aux->btf, info->type_id);
  447}
  448
  449static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog)
  450{
  451	struct bpf_subprog_info *info = subprog_info(env, subprog);
  452
  453	info->is_cb = true;
  454	info->is_async_cb = true;
  455	info->is_exception_cb = true;
  456}
  457
  458static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog)
  459{
  460	return subprog_info(env, subprog)->is_exception_cb;
  461}
  462
  463static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
  464{
  465	return btf_record_has_field(reg_btf_record(reg), BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK);
  466}
  467
  468static bool type_is_rdonly_mem(u32 type)
  469{
  470	return type & MEM_RDONLY;
  471}
  472
  473static bool is_acquire_function(enum bpf_func_id func_id,
  474				const struct bpf_map *map)
  475{
  476	enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC;
  477
  478	if (func_id == BPF_FUNC_sk_lookup_tcp ||
  479	    func_id == BPF_FUNC_sk_lookup_udp ||
  480	    func_id == BPF_FUNC_skc_lookup_tcp ||
  481	    func_id == BPF_FUNC_ringbuf_reserve ||
  482	    func_id == BPF_FUNC_kptr_xchg)
  483		return true;
  484
  485	if (func_id == BPF_FUNC_map_lookup_elem &&
  486	    (map_type == BPF_MAP_TYPE_SOCKMAP ||
  487	     map_type == BPF_MAP_TYPE_SOCKHASH))
  488		return true;
  489
  490	return false;
  491}
  492
  493static bool is_ptr_cast_function(enum bpf_func_id func_id)
  494{
  495	return func_id == BPF_FUNC_tcp_sock ||
  496		func_id == BPF_FUNC_sk_fullsock ||
  497		func_id == BPF_FUNC_skc_to_tcp_sock ||
  498		func_id == BPF_FUNC_skc_to_tcp6_sock ||
  499		func_id == BPF_FUNC_skc_to_udp6_sock ||
  500		func_id == BPF_FUNC_skc_to_mptcp_sock ||
  501		func_id == BPF_FUNC_skc_to_tcp_timewait_sock ||
  502		func_id == BPF_FUNC_skc_to_tcp_request_sock;
  503}
  504
  505static bool is_dynptr_ref_function(enum bpf_func_id func_id)
  506{
  507	return func_id == BPF_FUNC_dynptr_data;
  508}
  509
  510static bool is_sync_callback_calling_kfunc(u32 btf_id);
  511static bool is_async_callback_calling_kfunc(u32 btf_id);
  512static bool is_callback_calling_kfunc(u32 btf_id);
  513static bool is_bpf_throw_kfunc(struct bpf_insn *insn);
  514
  515static bool is_bpf_wq_set_callback_impl_kfunc(u32 btf_id);
  516static bool is_task_work_add_kfunc(u32 func_id);
  517
  518static bool is_sync_callback_calling_function(enum bpf_func_id func_id)
  519{
  520	return func_id == BPF_FUNC_for_each_map_elem ||
  521	       func_id == BPF_FUNC_find_vma ||
  522	       func_id == BPF_FUNC_loop ||
  523	       func_id == BPF_FUNC_user_ringbuf_drain;
  524}
  525
  526static bool is_async_callback_calling_function(enum bpf_func_id func_id)
  527{
  528	return func_id == BPF_FUNC_timer_set_callback;
  529}
  530
  531static bool is_callback_calling_function(enum bpf_func_id func_id)
  532{
  533	return is_sync_callback_calling_function(func_id) ||
  534	       is_async_callback_calling_function(func_id);
  535}
  536
  537static bool is_sync_callback_calling_insn(struct bpf_insn *insn)
  538{
  539	return (bpf_helper_call(insn) && is_sync_callback_calling_function(insn->imm)) ||
  540	       (bpf_pseudo_kfunc_call(insn) && is_sync_callback_calling_kfunc(insn->imm));
  541}
  542
  543static bool is_async_callback_calling_insn(struct bpf_insn *insn)
  544{
  545	return (bpf_helper_call(insn) && is_async_callback_calling_function(insn->imm)) ||
  546	       (bpf_pseudo_kfunc_call(insn) && is_async_callback_calling_kfunc(insn->imm));
  547}
  548
  549static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn *insn)
  550{
  551	/* bpf_timer callbacks are never sleepable. */
  552	if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_timer_set_callback)
  553		return false;
  554
  555	/* bpf_wq and bpf_task_work callbacks are always sleepable. */
  556	if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
  557	    (is_bpf_wq_set_callback_impl_kfunc(insn->imm) || is_task_work_add_kfunc(insn->imm)))
  558		return true;
  559
  560	verifier_bug(env, "unhandled async callback in is_async_cb_sleepable");
  561	return false;
  562}
  563
  564static bool is_may_goto_insn(struct bpf_insn *insn)
  565{
  566	return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO;
  567}
  568
  569static bool is_may_goto_insn_at(struct bpf_verifier_env *env, int insn_idx)
  570{
  571	return is_may_goto_insn(&env->prog->insnsi[insn_idx]);
  572}
  573
  574static bool is_storage_get_function(enum bpf_func_id func_id)
  575{
  576	return func_id == BPF_FUNC_sk_storage_get ||
  577	       func_id == BPF_FUNC_inode_storage_get ||
  578	       func_id == BPF_FUNC_task_storage_get ||
  579	       func_id == BPF_FUNC_cgrp_storage_get;
  580}
  581
  582static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id,
  583					const struct bpf_map *map)
  584{
  585	int ref_obj_uses = 0;
  586
  587	if (is_ptr_cast_function(func_id))
  588		ref_obj_uses++;
  589	if (is_acquire_function(func_id, map))
  590		ref_obj_uses++;
  591	if (is_dynptr_ref_function(func_id))
  592		ref_obj_uses++;
  593
  594	return ref_obj_uses > 1;
  595}
  596
  597static bool is_cmpxchg_insn(const struct bpf_insn *insn)
  598{
  599	return BPF_CLASS(insn->code) == BPF_STX &&
  600	       BPF_MODE(insn->code) == BPF_ATOMIC &&
  601	       insn->imm == BPF_CMPXCHG;
  602}
  603
  604static bool is_atomic_load_insn(const struct bpf_insn *insn)
  605{
  606	return BPF_CLASS(insn->code) == BPF_STX &&
  607	       BPF_MODE(insn->code) == BPF_ATOMIC &&
  608	       insn->imm == BPF_LOAD_ACQ;
  609}
  610
  611static bool is_atomic_fetch_insn(const struct bpf_insn *insn)
  612{
  613	return BPF_CLASS(insn->code) == BPF_STX &&
  614	       BPF_MODE(insn->code) == BPF_ATOMIC &&
  615	       (insn->imm & BPF_FETCH);
  616}
  617
  618static int __get_spi(s32 off)
  619{
  620	return (-off - 1) / BPF_REG_SIZE;
  621}
  622
  623static struct bpf_func_state *func(struct bpf_verifier_env *env,
  624				   const struct bpf_reg_state *reg)
  625{
  626	struct bpf_verifier_state *cur = env->cur_state;
  627
  628	return cur->frame[reg->frameno];
  629}
  630
  631static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots)
  632{
  633       int allocated_slots = state->allocated_stack / BPF_REG_SIZE;
  634
  635       /* We need to check that slots between [spi - nr_slots + 1, spi] are
  636	* within [0, allocated_stack).
  637	*
  638	* Please note that the spi grows downwards. For example, a dynptr
  639	* takes the size of two stack slots; the first slot will be at
  640	* spi and the second slot will be at spi - 1.
  641	*/
  642       return spi - nr_slots + 1 >= 0 && spi < allocated_slots;
  643}
  644
  645static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
  646			          const char *obj_kind, int nr_slots)
  647{
  648	int off, spi;
  649
  650	if (!tnum_is_const(reg->var_off)) {
  651		verbose(env, "%s has to be at a constant offset\n", obj_kind);
  652		return -EINVAL;
  653	}
  654
  655	off = reg->off + reg->var_off.value;
  656	if (off % BPF_REG_SIZE) {
  657		verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);
  658		return -EINVAL;
  659	}
  660
  661	spi = __get_spi(off);
  662	if (spi + 1 < nr_slots) {
  663		verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);
  664		return -EINVAL;
  665	}
  666
  667	if (!is_spi_bounds_valid(func(env, reg), spi, nr_slots))
  668		return -ERANGE;
  669	return spi;
  670}
  671
  672static int dynptr_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
  673{
  674	return stack_slot_obj_get_spi(env, reg, "dynptr", BPF_DYNPTR_NR_SLOTS);
  675}
  676
  677static int iter_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots)
  678{
  679	return stack_slot_obj_get_spi(env, reg, "iter", nr_slots);
  680}
  681
  682static int irq_flag_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
  683{
  684	return stack_slot_obj_get_spi(env, reg, "irq_flag", 1);
  685}
  686
  687static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type)
  688{
  689	switch (arg_type & DYNPTR_TYPE_FLAG_MASK) {
  690	case DYNPTR_TYPE_LOCAL:
  691		return BPF_DYNPTR_TYPE_LOCAL;
  692	case DYNPTR_TYPE_RINGBUF:
  693		return BPF_DYNPTR_TYPE_RINGBUF;
  694	case DYNPTR_TYPE_SKB:
  695		return BPF_DYNPTR_TYPE_SKB;
  696	case DYNPTR_TYPE_XDP:
  697		return BPF_DYNPTR_TYPE_XDP;
  698	case DYNPTR_TYPE_SKB_META:
  699		return BPF_DYNPTR_TYPE_SKB_META;
  700	case DYNPTR_TYPE_FILE:
  701		return BPF_DYNPTR_TYPE_FILE;
  702	default:
  703		return BPF_DYNPTR_TYPE_INVALID;
  704	}
  705}
  706
  707static enum bpf_type_flag get_dynptr_type_flag(enum bpf_dynptr_type type)
  708{
  709	switch (type) {
  710	case BPF_DYNPTR_TYPE_LOCAL:
  711		return DYNPTR_TYPE_LOCAL;
  712	case BPF_DYNPTR_TYPE_RINGBUF:
  713		return DYNPTR_TYPE_RINGBUF;
  714	case BPF_DYNPTR_TYPE_SKB:
  715		return DYNPTR_TYPE_SKB;
  716	case BPF_DYNPTR_TYPE_XDP:
  717		return DYNPTR_TYPE_XDP;
  718	case BPF_DYNPTR_TYPE_SKB_META:
  719		return DYNPTR_TYPE_SKB_META;
  720	case BPF_DYNPTR_TYPE_FILE:
  721		return DYNPTR_TYPE_FILE;
  722	default:
  723		return 0;
  724	}
  725}
  726
  727static bool dynptr_type_refcounted(enum bpf_dynptr_type type)
  728{
  729	return type == BPF_DYNPTR_TYPE_RINGBUF || type == BPF_DYNPTR_TYPE_FILE;
  730}
  731
  732static void __mark_dynptr_reg(struct bpf_reg_state *reg,
  733			      enum bpf_dynptr_type type,
  734			      bool first_slot, int dynptr_id);
  735
  736static void __mark_reg_not_init(const struct bpf_verifier_env *env,
  737				struct bpf_reg_state *reg);
  738
  739static void mark_dynptr_stack_regs(struct bpf_verifier_env *env,
  740				   struct bpf_reg_state *sreg1,
  741				   struct bpf_reg_state *sreg2,
  742				   enum bpf_dynptr_type type)
  743{
  744	int id = ++env->id_gen;
  745
  746	__mark_dynptr_reg(sreg1, type, true, id);
  747	__mark_dynptr_reg(sreg2, type, false, id);
  748}
  749
  750static void mark_dynptr_cb_reg(struct bpf_verifier_env *env,
  751			       struct bpf_reg_state *reg,
  752			       enum bpf_dynptr_type type)
  753{
  754	__mark_dynptr_reg(reg, type, true, ++env->id_gen);
  755}
  756
  757static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
  758				        struct bpf_func_state *state, int spi);
  759
  760static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
  761				   enum bpf_arg_type arg_type, int insn_idx, int clone_ref_obj_id)
  762{
  763	struct bpf_func_state *state = func(env, reg);
  764	enum bpf_dynptr_type type;
  765	int spi, i, err;
  766
  767	spi = dynptr_get_spi(env, reg);
  768	if (spi < 0)
  769		return spi;
  770
  771	/* We cannot assume both spi and spi - 1 belong to the same dynptr,
  772	 * hence we need to call destroy_if_dynptr_stack_slot twice for both,
  773	 * to ensure that for the following example:
  774	 *	[d1][d1][d2][d2]
  775	 * spi    3   2   1   0
  776	 * So marking spi = 2 should lead to destruction of both d1 and d2. In
  777	 * case they do belong to same dynptr, second call won't see slot_type
  778	 * as STACK_DYNPTR and will simply skip destruction.
  779	 */
  780	err = destroy_if_dynptr_stack_slot(env, state, spi);
  781	if (err)
  782		return err;
  783	err = destroy_if_dynptr_stack_slot(env, state, spi - 1);
  784	if (err)
  785		return err;
  786
  787	for (i = 0; i < BPF_REG_SIZE; i++) {
  788		state->stack[spi].slot_type[i] = STACK_DYNPTR;
  789		state->stack[spi - 1].slot_type[i] = STACK_DYNPTR;
  790	}
  791
  792	type = arg_to_dynptr_type(arg_type);
  793	if (type == BPF_DYNPTR_TYPE_INVALID)
  794		return -EINVAL;
  795
  796	mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr,
  797			       &state->stack[spi - 1].spilled_ptr, type);
  798
  799	if (dynptr_type_refcounted(type)) {
  800		/* The id is used to track proper releasing */
  801		int id;
  802
  803		if (clone_ref_obj_id)
  804			id = clone_ref_obj_id;
  805		else
  806			id = acquire_reference(env, insn_idx);
  807
  808		if (id < 0)
  809			return id;
  810
  811		state->stack[spi].spilled_ptr.ref_obj_id = id;
  812		state->stack[spi - 1].spilled_ptr.ref_obj_id = id;
  813	}
  814
  815	bpf_mark_stack_write(env, state->frameno, BIT(spi - 1) | BIT(spi));
  816
  817	return 0;
  818}
  819
  820static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi)
  821{
  822	int i;
  823
  824	for (i = 0; i < BPF_REG_SIZE; i++) {
  825		state->stack[spi].slot_type[i] = STACK_INVALID;
  826		state->stack[spi - 1].slot_type[i] = STACK_INVALID;
  827	}
  828
  829	__mark_reg_not_init(env, &state->stack[spi].spilled_ptr);
  830	__mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr);
  831
  832	bpf_mark_stack_write(env, state->frameno, BIT(spi - 1) | BIT(spi));
  833}
  834
  835static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
  836{
  837	struct bpf_func_state *state = func(env, reg);
  838	int spi, ref_obj_id, i;
  839
  840	/*
  841	 * This can only be set for PTR_TO_STACK, as CONST_PTR_TO_DYNPTR cannot
  842	 * be released by any dynptr helper. Hence, unmark_stack_slots_dynptr
  843	 * is safe to do directly.
  844	 */
  845	if (reg->type == CONST_PTR_TO_DYNPTR) {
  846		verifier_bug(env, "CONST_PTR_TO_DYNPTR cannot be released");
  847		return -EFAULT;
  848	}
  849	spi = dynptr_get_spi(env, reg);
  850	if (spi < 0)
  851		return spi;
  852
  853	if (!dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) {
  854		invalidate_dynptr(env, state, spi);
  855		return 0;
  856	}
  857
  858	ref_obj_id = state->stack[spi].spilled_ptr.ref_obj_id;
  859
  860	/* If the dynptr has a ref_obj_id, then we need to invalidate
  861	 * two things:
  862	 *
  863	 * 1) Any dynptrs with a matching ref_obj_id (clones)
  864	 * 2) Any slices derived from this dynptr.
  865	 */
  866
  867	/* Invalidate any slices associated with this dynptr */
  868	WARN_ON_ONCE(release_reference(env, ref_obj_id));
  869
  870	/* Invalidate any dynptr clones */
  871	for (i = 1; i < state->allocated_stack / BPF_REG_SIZE; i++) {
  872		if (state->stack[i].spilled_ptr.ref_obj_id != ref_obj_id)
  873			continue;
  874
  875		/* it should always be the case that if the ref obj id
  876		 * matches then the stack slot also belongs to a
  877		 * dynptr
  878		 */
  879		if (state->stack[i].slot_type[0] != STACK_DYNPTR) {
  880			verifier_bug(env, "misconfigured ref_obj_id");
  881			return -EFAULT;
  882		}
  883		if (state->stack[i].spilled_ptr.dynptr.first_slot)
  884			invalidate_dynptr(env, state, i);
  885	}
  886
  887	return 0;
  888}
  889
  890static void __mark_reg_unknown(const struct bpf_verifier_env *env,
  891			       struct bpf_reg_state *reg);
  892
  893static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)
  894{
  895	if (!env->allow_ptr_leaks)
  896		__mark_reg_not_init(env, reg);
  897	else
  898		__mark_reg_unknown(env, reg);
  899}
  900
  901static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
  902				        struct bpf_func_state *state, int spi)
  903{
  904	struct bpf_func_state *fstate;
  905	struct bpf_reg_state *dreg;
  906	int i, dynptr_id;
  907
  908	/* We always ensure that STACK_DYNPTR is never set partially,
  909	 * hence just checking for slot_type[0] is enough. This is
  910	 * different for STACK_SPILL, where it may be only set for
  911	 * 1 byte, so code has to use is_spilled_reg.
  912	 */
  913	if (state->stack[spi].slot_type[0] != STACK_DYNPTR)
  914		return 0;
  915
  916	/* Reposition spi to first slot */
  917	if (!state->stack[spi].spilled_ptr.dynptr.first_slot)
  918		spi = spi + 1;
  919
  920	if (dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) {
  921		verbose(env, "cannot overwrite referenced dynptr\n");
  922		return -EINVAL;
  923	}
  924
  925	mark_stack_slot_scratched(env, spi);
  926	mark_stack_slot_scratched(env, spi - 1);
  927
  928	/* Writing partially to one dynptr stack slot destroys both. */
  929	for (i = 0; i < BPF_REG_SIZE; i++) {
  930		state->stack[spi].slot_type[i] = STACK_INVALID;
  931		state->stack[spi - 1].slot_type[i] = STACK_INVALID;
  932	}
  933
  934	dynptr_id = state->stack[spi].spilled_ptr.id;
  935	/* Invalidate any slices associated with this dynptr */
  936	bpf_for_each_reg_in_vstate(env->cur_state, fstate, dreg, ({
  937		/* Dynptr slices are only PTR_TO_MEM_OR_NULL and PTR_TO_MEM */
  938		if (dreg->type != (PTR_TO_MEM | PTR_MAYBE_NULL) && dreg->type != PTR_TO_MEM)
  939			continue;
  940		if (dreg->dynptr_id == dynptr_id)
  941			mark_reg_invalid(env, dreg);
  942	}));
  943
  944	/* Do not release reference state, we are destroying dynptr on stack,
  945	 * not using some helper to release it. Just reset register.
  946	 */
  947	__mark_reg_not_init(env, &state->stack[spi].spilled_ptr);
  948	__mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr);
  949
  950	bpf_mark_stack_write(env, state->frameno, BIT(spi - 1) | BIT(spi));
  951
  952	return 0;
  953}
  954
  955static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
  956{
  957	int spi;
  958
  959	if (reg->type == CONST_PTR_TO_DYNPTR)
  960		return false;
  961
  962	spi = dynptr_get_spi(env, reg);
  963
  964	/* -ERANGE (i.e. spi not falling into allocated stack slots) isn't an
  965	 * error because this just means the stack state hasn't been updated yet.
  966	 * We will do check_mem_access to check and update stack bounds later.
  967	 */
  968	if (spi < 0 && spi != -ERANGE)
  969		return false;
  970
  971	/* We don't need to check if the stack slots are marked by previous
  972	 * dynptr initializations because we allow overwriting existing unreferenced
  973	 * STACK_DYNPTR slots, see mark_stack_slots_dynptr which calls
  974	 * destroy_if_dynptr_stack_slot to ensure dynptr objects at the slots we are
  975	 * touching are completely destructed before we reinitialize them for a new
  976	 * one. For referenced ones, destroy_if_dynptr_stack_slot returns an error early
  977	 * instead of delaying it until the end where the user will get "Unreleased
  978	 * reference" error.
  979	 */
  980	return true;
  981}
  982
  983static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
  984{
  985	struct bpf_func_state *state = func(env, reg);
  986	int i, spi;
  987
  988	/* This already represents first slot of initialized bpf_dynptr.
  989	 *
  990	 * CONST_PTR_TO_DYNPTR already has fixed and var_off as 0 due to
  991	 * check_func_arg_reg_off's logic, so we don't need to check its
  992	 * offset and alignment.
  993	 */
  994	if (reg->type == CONST_PTR_TO_DYNPTR)
  995		return true;
  996
  997	spi = dynptr_get_spi(env, reg);
  998	if (spi < 0)
  999		return false;
 1000	if (!state->stack[spi].spilled_ptr.dynptr.first_slot)
 1001		return false;
 1002
 1003	for (i = 0; i < BPF_REG_SIZE; i++) {
 1004		if (state->stack[spi].slot_type[i] != STACK_DYNPTR ||
 1005		    state->stack[spi - 1].slot_type[i] != STACK_DYNPTR)
 1006			return false;
 1007	}
 1008
 1009	return true;
 1010}
 1011
 1012static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
 1013				    enum bpf_arg_type arg_type)
 1014{
 1015	struct bpf_func_state *state = func(env, reg);
 1016	enum bpf_dynptr_type dynptr_type;
 1017	int spi;
 1018
 1019	/* ARG_PTR_TO_DYNPTR takes any type of dynptr */
 1020	if (arg_type == ARG_PTR_TO_DYNPTR)
 1021		return true;
 1022
 1023	dynptr_type = arg_to_dynptr_type(arg_type);
 1024	if (reg->type == CONST_PTR_TO_DYNPTR) {
 1025		return reg->dynptr.type == dynptr_type;
 1026	} else {
 1027		spi = dynptr_get_spi(env, reg);
 1028		if (spi < 0)
 1029			return false;
 1030		return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type;
 1031	}
 1032}
 1033
 1034static void __mark_reg_known_zero(struct bpf_reg_state *reg);
 1035
 1036static bool in_rcu_cs(struct bpf_verifier_env *env);
 1037
 1038static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta);
 1039
 1040static int mark_stack_slots_iter(struct bpf_verifier_env *env,
 1041				 struct bpf_kfunc_call_arg_meta *meta,
 1042				 struct bpf_reg_state *reg, int insn_idx,
 1043				 struct btf *btf, u32 btf_id, int nr_slots)
 1044{
 1045	struct bpf_func_state *state = func(env, reg);
 1046	int spi, i, j, id;
 1047
 1048	spi = iter_get_spi(env, reg, nr_slots);
 1049	if (spi < 0)
 1050		return spi;
 1051
 1052	id = acquire_reference(env, insn_idx);
 1053	if (id < 0)
 1054		return id;
 1055
 1056	for (i = 0; i < nr_slots; i++) {
 1057		struct bpf_stack_state *slot = &state->stack[spi - i];
 1058		struct bpf_reg_state *st = &slot->spilled_ptr;
 1059
 1060		__mark_reg_known_zero(st);
 1061		st->type = PTR_TO_STACK; /* we don't have dedicated reg type */
 1062		if (is_kfunc_rcu_protected(meta)) {
 1063			if (in_rcu_cs(env))
 1064				st->type |= MEM_RCU;
 1065			else
 1066				st->type |= PTR_UNTRUSTED;
 1067		}
 1068		st->ref_obj_id = i == 0 ? id : 0;
 1069		st->iter.btf = btf;
 1070		st->iter.btf_id = btf_id;
 1071		st->iter.state = BPF_ITER_STATE_ACTIVE;
 1072		st->iter.depth = 0;
 1073
 1074		for (j = 0; j < BPF_REG_SIZE; j++)
 1075			slot->slot_type[j] = STACK_ITER;
 1076
 1077		bpf_mark_stack_write(env, state->frameno, BIT(spi - i));
 1078		mark_stack_slot_scratched(env, spi - i);
 1079	}
 1080
 1081	return 0;
 1082}
 1083
 1084static int unmark_stack_slots_iter(struct bpf_verifier_env *env,
 1085				   struct bpf_reg_state *reg, int nr_slots)
 1086{
 1087	struct bpf_func_state *state = func(env, reg);
 1088	int spi, i, j;
 1089
 1090	spi = iter_get_spi(env, reg, nr_slots);
 1091	if (spi < 0)
 1092		return spi;
 1093
 1094	for (i = 0; i < nr_slots; i++) {
 1095		struct bpf_stack_state *slot = &state->stack[spi - i];
 1096		struct bpf_reg_state *st = &slot->spilled_ptr;
 1097
 1098		if (i == 0)
 1099			WARN_ON_ONCE(release_reference(env, st->ref_obj_id));
 1100
 1101		__mark_reg_not_init(env, st);
 1102
 1103		for (j = 0; j < BPF_REG_SIZE; j++)
 1104			slot->slot_type[j] = STACK_INVALID;
 1105
 1106		bpf_mark_stack_write(env, state->frameno, BIT(spi - i));
 1107		mark_stack_slot_scratched(env, spi - i);
 1108	}
 1109
 1110	return 0;
 1111}
 1112
 1113static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env,
 1114				     struct bpf_reg_state *reg, int nr_slots)
 1115{
 1116	struct bpf_func_state *state = func(env, reg);
 1117	int spi, i, j;
 1118
 1119	/* For -ERANGE (i.e. spi not falling into allocated stack slots), we
 1120	 * will do check_mem_access to check and update stack bounds later, so
 1121	 * return true for that case.
 1122	 */
 1123	spi = iter_get_spi(env, reg, nr_slots);
 1124	if (spi == -ERANGE)
 1125		return true;
 1126	if (spi < 0)
 1127		return false;
 1128
 1129	for (i = 0; i < nr_slots; i++) {
 1130		struct bpf_stack_state *slot = &state->stack[spi - i];
 1131
 1132		for (j = 0; j < BPF_REG_SIZE; j++)
 1133			if (slot->slot_type[j] == STACK_ITER)
 1134				return false;
 1135	}
 1136
 1137	return true;
 1138}
 1139
 1140static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
 1141				   struct btf *btf, u32 btf_id, int nr_slots)
 1142{
 1143	struct bpf_func_state *state = func(env, reg);
 1144	int spi, i, j;
 1145
 1146	spi = iter_get_spi(env, reg, nr_slots);
 1147	if (spi < 0)
 1148		return -EINVAL;
 1149
 1150	for (i = 0; i < nr_slots; i++) {
 1151		struct bpf_stack_state *slot = &state->stack[spi - i];
 1152		struct bpf_reg_state *st = &slot->spilled_ptr;
 1153
 1154		if (st->type & PTR_UNTRUSTED)
 1155			return -EPROTO;
 1156		/* only main (first) slot has ref_obj_id set */
 1157		if (i == 0 && !st->ref_obj_id)
 1158			return -EINVAL;
 1159		if (i != 0 && st->ref_obj_id)
 1160			return -EINVAL;
 1161		if (st->iter.btf != btf || st->iter.btf_id != btf_id)
 1162			return -EINVAL;
 1163
 1164		for (j = 0; j < BPF_REG_SIZE; j++)
 1165			if (slot->slot_type[j] != STACK_ITER)
 1166				return -EINVAL;
 1167	}
 1168
 1169	return 0;
 1170}
 1171
 1172static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx);
 1173static int release_irq_state(struct bpf_verifier_state *state, int id);
 1174
 1175static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env,
 1176				     struct bpf_kfunc_call_arg_meta *meta,
 1177				     struct bpf_reg_state *reg, int insn_idx,
 1178				     int kfunc_class)
 1179{
 1180	struct bpf_func_state *state = func(env, reg);
 1181	struct bpf_stack_state *slot;
 1182	struct bpf_reg_state *st;
 1183	int spi, i, id;
 1184
 1185	spi = irq_flag_get_spi(env, reg);
 1186	if (spi < 0)
 1187		return spi;
 1188
 1189	id = acquire_irq_state(env, insn_idx);
 1190	if (id < 0)
 1191		return id;
 1192
 1193	slot = &state->stack[spi];
 1194	st = &slot->spilled_ptr;
 1195
 1196	bpf_mark_stack_write(env, reg->frameno, BIT(spi));
 1197	__mark_reg_known_zero(st);
 1198	st->type = PTR_TO_STACK; /* we don't have dedicated reg type */
 1199	st->ref_obj_id = id;
 1200	st->irq.kfunc_class = kfunc_class;
 1201
 1202	for (i = 0; i < BPF_REG_SIZE; i++)
 1203		slot->slot_type[i] = STACK_IRQ_FLAG;
 1204
 1205	mark_stack_slot_scratched(env, spi);
 1206	return 0;
 1207}
 1208
 1209static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
 1210				      int kfunc_class)
 1211{
 1212	struct bpf_func_state *state = func(env, reg);
 1213	struct bpf_stack_state *slot;
 1214	struct bpf_reg_state *st;
 1215	int spi, i, err;
 1216
 1217	spi = irq_flag_get_spi(env, reg);
 1218	if (spi < 0)
 1219		return spi;
 1220
 1221	slot = &state->stack[spi];
 1222	st = &slot->spilled_ptr;
 1223
 1224	if (st->irq.kfunc_class != kfunc_class) {
 1225		const char *flag_kfunc = st->irq.kfunc_class == IRQ_NATIVE_KFUNC ? "native" : "lock";
 1226		const char *used_kfunc = kfunc_class == IRQ_NATIVE_KFUNC ? "native" : "lock";
 1227
 1228		verbose(env, "irq flag acquired by %s kfuncs cannot be restored with %s kfuncs\n",
 1229			flag_kfunc, used_kfunc);
 1230		return -EINVAL;
 1231	}
 1232
 1233	err = release_irq_state(env->cur_state, st->ref_obj_id);
 1234	WARN_ON_ONCE(err && err != -EACCES);
 1235	if (err) {
 1236		int insn_idx = 0;
 1237
 1238		for (int i = 0; i < env->cur_state->acquired_refs; i++) {
 1239			if (env->cur_state->refs[i].id == env->cur_state->active_irq_id) {
 1240				insn_idx = env->cur_state->refs[i].insn_idx;
 1241				break;
 1242			}
 1243		}
 1244
 1245		verbose(env, "cannot restore irq state out of order, expected id=%d acquired at insn_idx=%d\n",
 1246			env->cur_state->active_irq_id, insn_idx);
 1247		return err;
 1248	}
 1249
 1250	__mark_reg_not_init(env, st);
 1251
 1252	bpf_mark_stack_write(env, reg->frameno, BIT(spi));
 1253
 1254	for (i = 0; i < BPF_REG_SIZE; i++)
 1255		slot->slot_type[i] = STACK_INVALID;
 1256
 1257	mark_stack_slot_scratched(env, spi);
 1258	return 0;
 1259}
 1260
 1261static bool is_irq_flag_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
 1262{
 1263	struct bpf_func_state *state = func(env, reg);
 1264	struct bpf_stack_state *slot;
 1265	int spi, i;
 1266
 1267	/* For -ERANGE (i.e. spi not falling into allocated stack slots), we
 1268	 * will do check_mem_access to check and update stack bounds later, so
 1269	 * return true for that case.
 1270	 */
 1271	spi = irq_flag_get_spi(env, reg);
 1272	if (spi == -ERANGE)
 1273		return true;
 1274	if (spi < 0)
 1275		return false;
 1276
 1277	slot = &state->stack[spi];
 1278
 1279	for (i = 0; i < BPF_REG_SIZE; i++)
 1280		if (slot->slot_type[i] == STACK_IRQ_FLAG)
 1281			return false;
 1282	return true;
 1283}
 1284
 1285static int is_irq_flag_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
 1286{
 1287	struct bpf_func_state *state = func(env, reg);
 1288	struct bpf_stack_state *slot;
 1289	struct bpf_reg_state *st;
 1290	int spi, i;
 1291
 1292	spi = irq_flag_get_spi(env, reg);
 1293	if (spi < 0)
 1294		return -EINVAL;
 1295
 1296	slot = &state->stack[spi];
 1297	st = &slot->spilled_ptr;
 1298
 1299	if (!st->ref_obj_id)
 1300		return -EINVAL;
 1301
 1302	for (i = 0; i < BPF_REG_SIZE; i++)
 1303		if (slot->slot_type[i] != STACK_IRQ_FLAG)
 1304			return -EINVAL;
 1305	return 0;
 1306}
 1307
 1308/* Check if given stack slot is "special":
 1309 *   - spilled register state (STACK_SPILL);
 1310 *   - dynptr state (STACK_DYNPTR);
 1311 *   - iter state (STACK_ITER).
 1312 *   - irq flag state (STACK_IRQ_FLAG)
 1313 */
 1314static bool is_stack_slot_special(const struct bpf_stack_state *stack)
 1315{
 1316	enum bpf_stack_slot_type type = stack->slot_type[BPF_REG_SIZE - 1];
 1317
 1318	switch (type) {
 1319	case STACK_SPILL:
 1320	case STACK_DYNPTR:
 1321	case STACK_ITER:
 1322	case STACK_IRQ_FLAG:
 1323		return true;
 1324	case STACK_INVALID:
 1325	case STACK_MISC:
 1326	case STACK_ZERO:
 1327		return false;
 1328	default:
 1329		WARN_ONCE(1, "unknown stack slot type %d\n", type);
 1330		return true;
 1331	}
 1332}
 1333
 1334/* The reg state of a pointer or a bounded scalar was saved when
 1335 * it was spilled to the stack.
 1336 */
 1337static bool is_spilled_reg(const struct bpf_stack_state *stack)
 1338{
 1339	return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL;
 1340}
 1341
 1342static bool is_spilled_scalar_reg(const struct bpf_stack_state *stack)
 1343{
 1344	return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL &&
 1345	       stack->spilled_ptr.type == SCALAR_VALUE;
 1346}
 1347
 1348static bool is_spilled_scalar_reg64(const struct bpf_stack_state *stack)
 1349{
 1350	return stack->slot_type[0] == STACK_SPILL &&
 1351	       stack->spilled_ptr.type == SCALAR_VALUE;
 1352}
 1353
 1354/* Mark stack slot as STACK_MISC, unless it is already STACK_INVALID, in which
 1355 * case they are equivalent, or it's STACK_ZERO, in which case we preserve
 1356 * more precise STACK_ZERO.
 1357 * Regardless of allow_ptr_leaks setting (i.e., privileged or unprivileged
 1358 * mode), we won't promote STACK_INVALID to STACK_MISC. In privileged case it is
 1359 * unnecessary as both are considered equivalent when loading data and pruning,
 1360 * in case of unprivileged mode it will be incorrect to allow reads of invalid
 1361 * slots.
 1362 */
 1363static void mark_stack_slot_misc(struct bpf_verifier_env *env, u8 *stype)
 1364{
 1365	if (*stype == STACK_ZERO)
 1366		return;
 1367	if (*stype == STACK_INVALID)
 1368		return;
 1369	*stype = STACK_MISC;
 1370}
 1371
 1372static void scrub_spilled_slot(u8 *stype)
 1373{
 1374	if (*stype != STACK_INVALID)
 1375		*stype = STACK_MISC;
 1376}
 1377
 1378/* copy array src of length n * size bytes to dst. dst is reallocated if it's too
 1379 * small to hold src. This is different from krealloc since we don't want to preserve
 1380 * the contents of dst.
 1381 *
 1382 * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could
 1383 * not be allocated.
 1384 */
 1385static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags)
 1386{
 1387	size_t alloc_bytes;
 1388	void *orig = dst;
 1389	size_t bytes;
 1390
 1391	if (ZERO_OR_NULL_PTR(src))
 1392		goto out;
 1393
 1394	if (unlikely(check_mul_overflow(n, size, &bytes)))
 1395		return NULL;
 1396
 1397	alloc_bytes = max(ksize(orig), kmalloc_size_roundup(bytes));
 1398	dst = krealloc(orig, alloc_bytes, flags);
 1399	if (!dst) {
 1400		kfree(orig);
 1401		return NULL;
 1402	}
 1403
 1404	memcpy(dst, src, bytes);
 1405out:
 1406	return dst ? dst : ZERO_SIZE_PTR;
 1407}
 1408
 1409/* resize an array from old_n items to new_n items. the array is reallocated if it's too
 1410 * small to hold new_n items. new items are zeroed out if the array grows.
 1411 *
 1412 * Contrary to krealloc_array, does not free arr if new_n is zero.
 1413 */
 1414static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size)
 1415{
 1416	size_t alloc_size;
 1417	void *new_arr;
 1418
 1419	if (!new_n || old_n == new_n)
 1420		goto out;
 1421
 1422	alloc_size = kmalloc_size_roundup(size_mul(new_n, size));
 1423	new_arr = krealloc(arr, alloc_size, GFP_KERNEL_ACCOUNT);
 1424	if (!new_arr) {
 1425		kfree(arr);
 1426		return NULL;
 1427	}
 1428	arr = new_arr;
 1429
 1430	if (new_n > old_n)
 1431		memset(arr + old_n * size, 0, (new_n - old_n) * size);
 1432
 1433out:
 1434	return arr ? arr : ZERO_SIZE_PTR;
 1435}
 1436
 1437static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf_verifier_state *src)
 1438{
 1439	dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs,
 1440			       sizeof(struct bpf_reference_state), GFP_KERNEL_ACCOUNT);
 1441	if (!dst->refs)
 1442		return -ENOMEM;
 1443
 1444	dst->acquired_refs = src->acquired_refs;
 1445	dst->active_locks = src->active_locks;
 1446	dst->active_preempt_locks = src->active_preempt_locks;
 1447	dst->active_rcu_locks = src->active_rcu_locks;
 1448	dst->active_irq_id = src->active_irq_id;
 1449	dst->active_lock_id = src->active_lock_id;
 1450	dst->active_lock_ptr = src->active_lock_ptr;
 1451	return 0;
 1452}
 1453
 1454static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src)
 1455{
 1456	size_t n = src->allocated_stack / BPF_REG_SIZE;
 1457
 1458	dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state),
 1459				GFP_KERNEL_ACCOUNT);
 1460	if (!dst->stack)
 1461		return -ENOMEM;
 1462
 1463	dst->allocated_stack = src->allocated_stack;
 1464	return 0;
 1465}
 1466
 1467static int resize_reference_state(struct bpf_verifier_state *state, size_t n)
 1468{
 1469	state->refs = realloc_array(state->refs, state->acquired_refs, n,
 1470				    sizeof(struct bpf_reference_state));
 1471	if (!state->refs)
 1472		return -ENOMEM;
 1473
 1474	state->acquired_refs = n;
 1475	return 0;
 1476}
 1477
 1478/* Possibly update state->allocated_stack to be at least size bytes. Also
 1479 * possibly update the function's high-water mark in its bpf_subprog_info.
 1480 */
 1481static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size)
 1482{
 1483	size_t old_n = state->allocated_stack / BPF_REG_SIZE, n;
 1484
 1485	/* The stack size is always a multiple of BPF_REG_SIZE. */
 1486	size = round_up(size, BPF_REG_SIZE);
 1487	n = size / BPF_REG_SIZE;
 1488
 1489	if (old_n >= n)
 1490		return 0;
 1491
 1492	state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state));
 1493	if (!state->stack)
 1494		return -ENOMEM;
 1495
 1496	state->allocated_stack = size;
 1497
 1498	/* update known max for given subprogram */
 1499	if (env->subprog_info[state->subprogno].stack_depth < size)
 1500		env->subprog_info[state->subprogno].stack_depth = size;
 1501
 1502	return 0;
 1503}
 1504
 1505/* Acquire a pointer id from the env and update the state->refs to include
 1506 * this new pointer reference.
 1507 * On success, returns a valid pointer id to associate with the register
 1508 * On failure, returns a negative errno.
 1509 */
 1510static struct bpf_reference_state *acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)
 1511{
 1512	struct bpf_verifier_state *state = env->cur_state;
 1513	int new_ofs = state->acquired_refs;
 1514	int err;
 1515
 1516	err = resize_reference_state(state, state->acquired_refs + 1);
 1517	if (err)
 1518		return NULL;
 1519	state->refs[new_ofs].insn_idx = insn_idx;
 1520
 1521	return &state->refs[new_ofs];
 1522}
 1523
 1524static int acquire_reference(struct bpf_verifier_env *env, int insn_idx)
 1525{
 1526	struct bpf_reference_state *s;
 1527
 1528	s = acquire_reference_state(env, insn_idx);
 1529	if (!s)
 1530		return -ENOMEM;
 1531	s->type = REF_TYPE_PTR;
 1532	s->id = ++env->id_gen;
 1533	return s->id;
 1534}
 1535
 1536static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum ref_state_type type,
 1537			      int id, void *ptr)
 1538{
 1539	struct bpf_verifier_state *state = env->cur_state;
 1540	struct bpf_reference_state *s;
 1541
 1542	s = acquire_reference_state(env, insn_idx);
 1543	if (!s)
 1544		return -ENOMEM;
 1545	s->type = type;
 1546	s->id = id;
 1547	s->ptr = ptr;
 1548
 1549	state->active_locks++;
 1550	state->active_lock_id = id;
 1551	state->active_lock_ptr = ptr;
 1552	return 0;
 1553}
 1554
 1555static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx)
 1556{
 1557	struct bpf_verifier_state *state = env->cur_state;
 1558	struct bpf_reference_state *s;
 1559
 1560	s = acquire_reference_state(env, insn_idx);
 1561	if (!s)
 1562		return -ENOMEM;
 1563	s->type = REF_TYPE_IRQ;
 1564	s->id = ++env->id_gen;
 1565
 1566	state->active_irq_id = s->id;
 1567	return s->id;
 1568}
 1569
 1570static void release_reference_state(struct bpf_verifier_state *state, int idx)
 1571{
 1572	int last_idx;
 1573	size_t rem;
 1574
 1575	/* IRQ state requires the relative ordering of elements remaining the
 1576	 * same, since it relies on the refs array to behave as a stack, so that
 1577	 * it can detect out-of-order IRQ restore. Hence use memmove to shift
 1578	 * the array instead of swapping the final element into the deleted idx.
 1579	 */
 1580	last_idx = state->acquired_refs - 1;
 1581	rem = state->acquired_refs - idx - 1;
 1582	if (last_idx && idx != last_idx)
 1583		memmove(&state->refs[idx], &state->refs[idx + 1], sizeof(*state->refs) * rem);
 1584	memset(&state->refs[last_idx], 0, sizeof(*state->refs));
 1585	state->acquired_refs--;
 1586	return;
 1587}
 1588
 1589static bool find_reference_state(struct bpf_verifier_state *state, int ptr_id)
 1590{
 1591	int i;
 1592
 1593	for (i = 0; i < state->acquired_refs; i++)
 1594		if (state->refs[i].id == ptr_id)
 1595			return true;
 1596
 1597	return false;
 1598}
 1599
 1600static int release_lock_state(struct bpf_verifier_state *state, int type, int id, void *ptr)
 1601{
 1602	void *prev_ptr = NULL;
 1603	u32 prev_id = 0;
 1604	int i;
 1605
 1606	for (i = 0; i < state->acquired_refs; i++) {
 1607		if (state->refs[i].type == type && state->refs[i].id == id &&
 1608		    state->refs[i].ptr == ptr) {
 1609			release_reference_state(state, i);
 1610			state->active_locks--;
 1611			/* Reassign active lock (id, ptr). */
 1612			state->active_lock_id = prev_id;
 1613			state->active_lock_ptr = prev_ptr;
 1614			return 0;
 1615		}
 1616		if (state->refs[i].type & REF_TYPE_LOCK_MASK) {
 1617			prev_id = state->refs[i].id;
 1618			prev_ptr = state->refs[i].ptr;
 1619		}
 1620	}
 1621	return -EINVAL;
 1622}
 1623
 1624static int release_irq_state(struct bpf_verifier_state *state, int id)
 1625{
 1626	u32 prev_id = 0;
 1627	int i;
 1628
 1629	if (id != state->active_irq_id)
 1630		return -EACCES;
 1631
 1632	for (i = 0; i < state->acquired_refs; i++) {
 1633		if (state->refs[i].type != REF_TYPE_IRQ)
 1634			continue;
 1635		if (state->refs[i].id == id) {
 1636			release_reference_state(state, i);
 1637			state->active_irq_id = prev_id;
 1638			return 0;
 1639		} else {
 1640			prev_id = state->refs[i].id;
 1641		}
 1642	}
 1643	return -EINVAL;
 1644}
 1645
 1646static struct bpf_reference_state *find_lock_state(struct bpf_verifier_state *state, enum ref_state_type type,
 1647						   int id, void *ptr)
 1648{
 1649	int i;
 1650
 1651	for (i = 0; i < state->acquired_refs; i++) {
 1652		struct bpf_reference_state *s = &state->refs[i];
 1653
 1654		if (!(s->type & type))
 1655			continue;
 1656
 1657		if (s->id == id && s->ptr == ptr)
 1658			return s;
 1659	}
 1660	return NULL;
 1661}
 1662
 1663static void update_peak_states(struct bpf_verifier_env *env)
 1664{
 1665	u32 cur_states;
 1666
 1667	cur_states = env->explored_states_size + env->free_list_size + env->num_backedges;
 1668	env->peak_states = max(env->peak_states, cur_states);
 1669}
 1670
 1671static void free_func_state(struct bpf_func_state *state)
 1672{
 1673	if (!state)
 1674		return;
 1675	kfree(state->stack);
 1676	kfree(state);
 1677}
 1678
 1679static void clear_jmp_history(struct bpf_verifier_state *state)
 1680{
 1681	kfree(state->jmp_history);
 1682	state->jmp_history = NULL;
 1683	state->jmp_history_cnt = 0;
 1684}
 1685
 1686static void free_verifier_state(struct bpf_verifier_state *state,
 1687				bool free_self)
 1688{
 1689	int i;
 1690
 1691	for (i = 0; i <= state->curframe; i++) {
 1692		free_func_state(state->frame[i]);
 1693		state->frame[i] = NULL;
 1694	}
 1695	kfree(state->refs);
 1696	clear_jmp_history(state);
 1697	if (free_self)
 1698		kfree(state);
 1699}
 1700
 1701/* struct bpf_verifier_state->parent refers to states
 1702 * that are in either of env->{expored_states,free_list}.
 1703 * In both cases the state is contained in struct bpf_verifier_state_list.
 1704 */
 1705static struct bpf_verifier_state_list *state_parent_as_list(struct bpf_verifier_state *st)
 1706{
 1707	if (st->parent)
 1708		return container_of(st->parent, struct bpf_verifier_state_list, state);
 1709	return NULL;
 1710}
 1711
 1712static bool incomplete_read_marks(struct bpf_verifier_env *env,
 1713				  struct bpf_verifier_state *st);
 1714
 1715/* A state can be freed if it is no longer referenced:
 1716 * - is in the env->free_list;
 1717 * - has no children states;
 1718 */
 1719static void maybe_free_verifier_state(struct bpf_verifier_env *env,
 1720				      struct bpf_verifier_state_list *sl)
 1721{
 1722	if (!sl->in_free_list
 1723	    || sl->state.branches != 0
 1724	    || incomplete_read_marks(env, &sl->state))
 1725		return;
 1726	list_del(&sl->node);
 1727	free_verifier_state(&sl->state, false);
 1728	kfree(sl);
 1729	env->free_list_size--;
 1730}
 1731
 1732/* copy verifier state from src to dst growing dst stack space
 1733 * when necessary to accommodate larger src stack
 1734 */
 1735static int copy_func_state(struct bpf_func_state *dst,
 1736			   const struct bpf_func_state *src)
 1737{
 1738	memcpy(dst, src, offsetof(struct bpf_func_state, stack));
 1739	return copy_stack_state(dst, src);
 1740}
 1741
 1742static int copy_verifier_state(struct bpf_verifier_state *dst_state,
 1743			       const struct bpf_verifier_state *src)
 1744{
 1745	struct bpf_func_state *dst;
 1746	int i, err;
 1747
 1748	dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history,
 1749					  src->jmp_history_cnt, sizeof(*dst_state->jmp_history),
 1750					  GFP_KERNEL_ACCOUNT);
 1751	if (!dst_state->jmp_history)
 1752		return -ENOMEM;
 1753	dst_state->jmp_history_cnt = src->jmp_history_cnt;
 1754
 1755	/* if dst has more stack frames then src frame, free them, this is also
 1756	 * necessary in case of exceptional exits using bpf_throw.
 1757	 */
 1758	for (i = src->curframe + 1; i <= dst_state->curframe; i++) {
 1759		free_func_state(dst_state->frame[i]);
 1760		dst_state->frame[i] = NULL;
 1761	}
 1762	err = copy_reference_state(dst_state, src);
 1763	if (err)
 1764		return err;
 1765	dst_state->speculative = src->speculative;
 1766	dst_state->in_sleepable = src->in_sleepable;
 1767	dst_state->cleaned = src->cleaned;
 1768	dst_state->curframe = src->curframe;
 1769	dst_state->branches = src->branches;
 1770	dst_state->parent = src->parent;
 1771	dst_state->first_insn_idx = src->first_insn_idx;
 1772	dst_state->last_insn_idx = src->last_insn_idx;
 1773	dst_state->dfs_depth = src->dfs_depth;
 1774	dst_state->callback_unroll_depth = src->callback_unroll_depth;
 1775	dst_state->may_goto_depth = src->may_goto_depth;
 1776	dst_state->equal_state = src->equal_state;
 1777	for (i = 0; i <= src->curframe; i++) {
 1778		dst = dst_state->frame[i];
 1779		if (!dst) {
 1780			dst = kzalloc(sizeof(*dst), GFP_KERNEL_ACCOUNT);
 1781			if (!dst)
 1782				return -ENOMEM;
 1783			dst_state->frame[i] = dst;
 1784		}
 1785		err = copy_func_state(dst, src->frame[i]);
 1786		if (err)
 1787			return err;
 1788	}
 1789	return 0;
 1790}
 1791
 1792static u32 state_htab_size(struct bpf_verifier_env *env)
 1793{
 1794	return env->prog->len;
 1795}
 1796
 1797static struct list_head *explored_state(struct bpf_verifier_env *env, int idx)
 1798{
 1799	struct bpf_verifier_state *cur = env->cur_state;
 1800	struct bpf_func_state *state = cur->frame[cur->curframe];
 1801
 1802	return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)];
 1803}
 1804
 1805static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_state *b)
 1806{
 1807	int fr;
 1808
 1809	if (a->curframe != b->curframe)
 1810		return false;
 1811
 1812	for (fr = a->curframe; fr >= 0; fr--)
 1813		if (a->frame[fr]->callsite != b->frame[fr]->callsite)
 1814			return false;
 1815
 1816	return true;
 1817}
 1818
 1819/* Return IP for a given frame in a call stack */
 1820static u32 frame_insn_idx(struct bpf_verifier_state *st, u32 frame)
 1821{
 1822	return frame == st->curframe
 1823	       ? st->insn_idx
 1824	       : st->frame[frame + 1]->callsite;
 1825}
 1826
 1827/* For state @st look for a topmost frame with frame_insn_idx() in some SCC,
 1828 * if such frame exists form a corresponding @callchain as an array of
 1829 * call sites leading to this frame and SCC id.
 1830 * E.g.:
 1831 *
 1832 *    void foo()  { A: loop {... SCC#1 ...}; }
 1833 *    void bar()  { B: loop { C: foo(); ... SCC#2 ... }
 1834 *                  D: loop { E: foo(); ... SCC#3 ... } }
 1835 *    void main() { F: bar(); }
 1836 *
 1837 * @callchain at (A) would be either (F,SCC#2) or (F,SCC#3) depending
 1838 * on @st frame call sites being (F,C,A) or (F,E,A).
 1839 */
 1840static bool compute_scc_callchain(struct bpf_verifier_env *env,
 1841				  struct bpf_verifier_state *st,
 1842				  struct bpf_scc_callchain *callchain)
 1843{
 1844	u32 i, scc, insn_idx;
 1845
 1846	memset(callchain, 0, sizeof(*callchain));
 1847	for (i = 0; i <= st->curframe; i++) {
 1848		insn_idx = frame_insn_idx(st, i);
 1849		scc = env->insn_aux_data[insn_idx].scc;
 1850		if (scc) {
 1851			callchain->scc = scc;
 1852			break;
 1853		} else if (i < st->curframe) {
 1854			callchain->callsites[i] = insn_idx;
 1855		} else {
 1856			return false;
 1857		}
 1858	}
 1859	return true;
 1860}
 1861
 1862/* Check if bpf_scc_visit instance for @callchain exists. */
 1863static struct bpf_scc_visit *scc_visit_lookup(struct bpf_verifier_env *env,
 1864					      struct bpf_scc_callchain *callchain)
 1865{
 1866	struct bpf_scc_info *info = env->scc_info[callchain->scc];
 1867	struct bpf_scc_visit *visits = info->visits;
 1868	u32 i;
 1869
 1870	if (!info)
 1871		return NULL;
 1872	for (i = 0; i < info->num_visits; i++)
 1873		if (memcmp(callchain, &visits[i].callchain, sizeof(*callchain)) == 0)
 1874			return &visits[i];
 1875	return NULL;
 1876}
 1877
 1878/* Allocate a new bpf_scc_visit instance corresponding to @callchain.
 1879 * Allocated instances are alive for a duration of the do_check_common()
 1880 * call and are freed by free_states().
 1881 */
 1882static struct bpf_scc_visit *scc_visit_alloc(struct bpf_verifier_env *env,
 1883					     struct bpf_scc_callchain *callchain)
 1884{
 1885	struct bpf_scc_visit *visit;
 1886	struct bpf_scc_info *info;
 1887	u32 scc, num_visits;
 1888	u64 new_sz;
 1889
 1890	scc = callchain->scc;
 1891	info = env->scc_info[scc];
 1892	num_visits = info ? info->num_visits : 0;
 1893	new_sz = sizeof(*info) + sizeof(struct bpf_scc_visit) * (num_visits + 1);
 1894	info = kvrealloc(env->scc_info[scc], new_sz, GFP_KERNEL_ACCOUNT);
 1895	if (!info)
 1896		return NULL;
 1897	env->scc_info[scc] = info;
 1898	info->num_visits = num_visits + 1;
 1899	visit = &info->visits[num_visits];
 1900	memset(visit, 0, sizeof(*visit));
 1901	memcpy(&visit->callchain, callchain, sizeof(*callchain));
 1902	return visit;
 1903}
 1904
 1905/* Form a string '(callsite#1,callsite#2,...,scc)' in env->tmp_str_buf */
 1906static char *format_callchain(struct bpf_verifier_env *env, struct bpf_scc_callchain *callchain)
 1907{
 1908	char *buf = env->tmp_str_buf;
 1909	int i, delta = 0;
 1910
 1911	delta += snprintf(buf + delta, TMP_STR_BUF_LEN - delta, "(");
 1912	for (i = 0; i < ARRAY_SIZE(callchain->callsites); i++) {
 1913		if (!callchain->callsites[i])
 1914			break;
 1915		delta += snprintf(buf + delta, TMP_STR_BUF_LEN - delta, "%u,",
 1916				  callchain->callsites[i]);
 1917	}
 1918	delta += snprintf(buf + delta, TMP_STR_BUF_LEN - delta, "%u)", callchain->scc);
 1919	return env->tmp_str_buf;
 1920}
 1921
 1922/* If callchain for @st exists (@st is in some SCC), ensure that
 1923 * bpf_scc_visit instance for this callchain exists.
 1924 * If instance does not exist or is empty, assign visit->entry_state to @st.
 1925 */
 1926static int maybe_enter_scc(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
 1927{
 1928	struct bpf_scc_callchain *callchain = &env->callchain_buf;
 1929	struct bpf_scc_visit *visit;
 1930
 1931	if (!compute_scc_callchain(env, st, callchain))
 1932		return 0;
 1933	visit = scc_visit_lookup(env, callchain);
 1934	visit = visit ?: scc_visit_alloc(env, callchain);
 1935	if (!visit)
 1936		return -ENOMEM;
 1937	if (!visit->entry_state) {
 1938		visit->entry_state = st;
 1939		if (env->log.level & BPF_LOG_LEVEL2)
 1940			verbose(env, "SCC enter %s\n", format_callchain(env, callchain));
 1941	}
 1942	return 0;
 1943}
 1944
 1945static int propagate_backedges(struct bpf_verifier_env *env, struct bpf_scc_visit *visit);
 1946
 1947/* If callchain for @st exists (@st is in some SCC), make it empty:
 1948 * - set visit->entry_state to NULL;
 1949 * - flush accumulated backedges.
 1950 */
 1951static int maybe_exit_scc(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
 1952{
 1953	struct bpf_scc_callchain *callchain = &env->callchain_buf;
 1954	struct bpf_scc_visit *visit;
 1955
 1956	if (!compute_scc_callchain(env, st, callchain))
 1957		return 0;
 1958	visit = scc_visit_lookup(env, callchain);
 1959	if (!visit) {
 1960		/*
 1961		 * If path traversal stops inside an SCC, corresponding bpf_scc_visit
 1962		 * must exist for non-speculative paths. For non-speculative paths
 1963		 * traversal stops when:
 1964		 * a. Verification error is found, maybe_exit_scc() is not called.
 1965		 * b. Top level BPF_EXIT is reached. Top level BPF_EXIT is not a member
 1966		 *    of any SCC.
 1967		 * c. A checkpoint is reached and matched. Checkpoints are created by
 1968		 *    is_state_visited(), which calls maybe_enter_scc(), which allocates
 1969		 *    bpf_scc_visit instances for checkpoints within SCCs.
 1970		 * (c) is the only case that can reach this point.
 1971		 */
 1972		if (!st->speculative) {
 1973			verifier_bug(env, "scc exit: no visit info for call chain %s",
 1974				     format_callchain(env, callchain));
 1975			return -EFAULT;
 1976		}
 1977		return 0;
 1978	}
 1979	if (visit->entry_state != st)
 1980		return 0;
 1981	if (env->log.level & BPF_LOG_LEVEL2)
 1982		verbose(env, "SCC exit %s\n", format_callchain(env, callchain));
 1983	visit->entry_state = NULL;
 1984	env->num_backedges -= visit->num_backedges;
 1985	visit->num_backedges = 0;
 1986	update_peak_states(env);
 1987	return propagate_backedges(env, visit);
 1988}
 1989
 1990/* Lookup an bpf_scc_visit instance corresponding to @st callchain
 1991 * and add @backedge to visit->backedges. @st callchain must exist.
 1992 */
 1993static int add_scc_backedge(struct bpf_verifier_env *env,
 1994			    struct bpf_verifier_state *st,
 1995			    struct bpf_scc_backedge *backedge)
 1996{
 1997	struct bpf_scc_callchain *callchain = &env->callchain_buf;
 1998	struct bpf_scc_visit *visit;
 1999
 2000	if (!compute_scc_callchain(env, st, callchain)) {
 2001		verifier_bug(env, "add backedge: no SCC in verification path, insn_idx %d",
 2002			     st->insn_idx);
 2003		return -EFAULT;
 2004	}
 2005	visit = scc_visit_lookup(env, callchain);
 2006	if (!visit) {
 2007		verifier_bug(env, "add backedge: no visit info for call chain %s",
 2008			     format_callchain(env, callchain));
 2009		return -EFAULT;
 2010	}
 2011	if (env->log.level & BPF_LOG_LEVEL2)
 2012		verbose(env, "SCC backedge %s\n", format_callchain(env, callchain));
 2013	backedge->next = visit->backedges;
 2014	visit->backedges = backedge;
 2015	visit->num_backedges++;
 2016	env->num_backedges++;
 2017	update_peak_states(env);
 2018	return 0;
 2019}
 2020
 2021/* bpf_reg_state->live marks for registers in a state @st are incomplete,
 2022 * if state @st is in some SCC and not all execution paths starting at this
 2023 * SCC are fully explored.
 2024 */
 2025static bool incomplete_read_marks(struct bpf_verifier_env *env,
 2026				  struct bpf_verifier_state *st)
 2027{
 2028	struct bpf_scc_callchain *callchain = &env->callchain_buf;
 2029	struct bpf_scc_visit *visit;
 2030
 2031	if (!compute_scc_callchain(env, st, callchain))
 2032		return false;
 2033	visit = scc_visit_lookup(env, callchain);
 2034	if (!visit)
 2035		return false;
 2036	return !!visit->backedges;
 2037}
 2038
 2039static void free_backedges(struct bpf_scc_visit *visit)
 2040{
 2041	struct bpf_scc_backedge *backedge, *next;
 2042
 2043	for (backedge = visit->backedges; backedge; backedge = next) {
 2044		free_verifier_state(&backedge->state, false);
 2045		next = backedge->next;
 2046		kfree(backedge);
 2047	}
 2048	visit->backedges = NULL;
 2049}
 2050
 2051static int update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
 2052{
 2053	struct bpf_verifier_state_list *sl = NULL, *parent_sl;
 2054	struct bpf_verifier_state *parent;
 2055	int err;
 2056
 2057	while (st) {
 2058		u32 br = --st->branches;
 2059
 2060		/* verifier_bug_if(br > 1, ...) technically makes sense here,
 2061		 * but see comment in push_stack(), hence:
 2062		 */
 2063		verifier_bug_if((int)br < 0, env, "%s:branches_to_explore=%d", __func__, br);
 2064		if (br)
 2065			break;
 2066		err = maybe_exit_scc(env, st);
 2067		if (err)
 2068			return err;
 2069		parent = st->parent;
 2070		parent_sl = state_parent_as_list(st);
 2071		if (sl)
 2072			maybe_free_verifier_state(env, sl);
 2073		st = parent;
 2074		sl = parent_sl;
 2075	}
 2076	return 0;
 2077}
 2078
 2079static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,
 2080		     int *insn_idx, bool pop_log)
 2081{
 2082	struct bpf_verifier_state *cur = env->cur_state;
 2083	struct bpf_verifier_stack_elem *elem, *head = env->head;
 2084	int err;
 2085
 2086	if (env->head == NULL)
 2087		return -ENOENT;
 2088
 2089	if (cur) {
 2090		err = copy_verifier_state(cur, &head->st);
 2091		if (err)
 2092			return err;
 2093	}
 2094	if (pop_log)
 2095		bpf_vlog_reset(&env->log, head->log_pos);
 2096	if (insn_idx)
 2097		*insn_idx = head->insn_idx;
 2098	if (prev_insn_idx)
 2099		*prev_insn_idx = head->prev_insn_idx;
 2100	elem = head->next;
 2101	free_verifier_state(&head->st, false);
 2102	kfree(head);
 2103	env->head = elem;
 2104	env->stack_size--;
 2105	return 0;
 2106}
 2107
 2108static bool error_recoverable_with_nospec(int err)
 2109{
 2110	/* Should only return true for non-fatal errors that are allowed to
 2111	 * occur during speculative verification. For these we can insert a
 2112	 * nospec and the program might still be accepted. Do not include
 2113	 * something like ENOMEM because it is likely to re-occur for the next
 2114	 * architectural path once it has been recovered-from in all speculative
 2115	 * paths.
 2116	 */
 2117	return err == -EPERM || err == -EACCES || err == -EINVAL;
 2118}
 2119
 2120static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
 2121					     int insn_idx, int prev_insn_idx,
 2122					     bool speculative)
 2123{
 2124	struct bpf_verifier_state *cur = env->cur_state;
 2125	struct bpf_verifier_stack_elem *elem;
 2126	int err;
 2127
 2128	elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL_ACCOUNT);
 2129	if (!elem)
 2130		return ERR_PTR(-ENOMEM);
 2131
 2132	elem->insn_idx = insn_idx;
 2133	elem->prev_insn_idx = prev_insn_idx;
 2134	elem->next = env->head;
 2135	elem->log_pos = env->log.end_pos;
 2136	env->head = elem;
 2137	env->stack_size++;
 2138	err = copy_verifier_state(&elem->st, cur);
 2139	if (err)
 2140		return ERR_PTR(-ENOMEM);
 2141	elem->st.speculative |= speculative;
 2142	if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
 2143		verbose(env, "The sequence of %d jumps is too complex.\n",
 2144			env->stack_size);
 2145		return ERR_PTR(-E2BIG);
 2146	}
 2147	if (elem->st.parent) {
 2148		++elem->st.parent->branches;
 2149		/* WARN_ON(branches > 2) technically makes sense here,
 2150		 * but
 2151		 * 1. speculative states will bump 'branches' for non-branch
 2152		 * instructions
 2153		 * 2. is_state_visited() heuristics may decide not to create
 2154		 * a new state for a sequence of branches and all such current
 2155		 * and cloned states will be pointing to a single parent state
 2156		 * which might have large 'branches' count.
 2157		 */
 2158	}
 2159	return &elem->st;
 2160}
 2161
 2162#define CALLER_SAVED_REGS 6
 2163static const int caller_saved[CALLER_SAVED_REGS] = {
 2164	BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5
 2165};
 2166
 2167/* This helper doesn't clear reg->id */
 2168static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm)
 2169{
 2170	reg->var_off = tnum_const(imm);
 2171	reg->smin_value = (s64)imm;
 2172	reg->smax_value = (s64)imm;
 2173	reg->umin_value = imm;
 2174	reg->umax_value = imm;
 2175
 2176	reg->s32_min_value = (s32)imm;
 2177	reg->s32_max_value = (s32)imm;
 2178	reg->u32_min_value = (u32)imm;
 2179	reg->u32_max_value = (u32)imm;
 2180}
 2181
 2182/* Mark the unknown part of a register (variable offset or scalar value) as
 2183 * known to have the value @imm.
 2184 */
 2185static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)
 2186{
 2187	/* Clear off and union(map_ptr, range) */
 2188	memset(((u8 *)reg) + sizeof(reg->type), 0,
 2189	       offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));
 2190	reg->id = 0;
 2191	reg->ref_obj_id = 0;
 2192	___mark_reg_known(reg, imm);
 2193}
 2194
 2195static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm)
 2196{
 2197	reg->var_off = tnum_const_subreg(reg->var_off, imm);
 2198	reg->s32_min_value = (s32)imm;
 2199	reg->s32_max_value = (s32)imm;
 2200	reg->u32_min_value = (u32)imm;
 2201	reg->u32_max_value = (u32)imm;
 2202}
 2203
 2204/* Mark the 'variable offset' part of a register as zero.  This should be
 2205 * used only on registers holding a pointer type.
 2206 */
 2207static void __mark_reg_known_zero(struct bpf_reg_state *reg)
 2208{
 2209	__mark_reg_known(reg, 0);
 2210}
 2211
 2212static void __mark_reg_const_zero(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)
 2213{
 2214	__mark_reg_known(reg, 0);
 2215	reg->type = SCALAR_VALUE;
 2216	/* all scalars are assumed imprecise initially (unless unprivileged,
 2217	 * in which case everything is forced to be precise)
 2218	 */
 2219	reg->precise = !env->bpf_capable;
 2220}
 2221
 2222static void mark_reg_known_zero(struct bpf_verifier_env *env,
 2223				struct bpf_reg_state *regs, u32 regno)
 2224{
 2225	if (WARN_ON(regno >= MAX_BPF_REG)) {
 2226		verbose(env, "mark_reg_known_zero(regs, %u)\n", regno);
 2227		/* Something bad happened, let's kill all regs */
 2228		for (regno = 0; regno < MAX_BPF_REG; regno++)
 2229			__mark_reg_not_init(env, regs + regno);
 2230		return;
 2231	}
 2232	__mark_reg_known_zero(regs + regno);
 2233}
 2234
 2235static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type,
 2236			      bool first_slot, int dynptr_id)
 2237{
 2238	/* reg->type has no meaning for STACK_DYNPTR, but when we set reg for
 2239	 * callback arguments, it does need to be CONST_PTR_TO_DYNPTR, so simply
 2240	 * set it unconditionally as it is ignored for STACK_DYNPTR anyway.
 2241	 */
 2242	__mark_reg_known_zero(reg);
 2243	reg->type = CONST_PTR_TO_DYNPTR;
 2244	/* Give each dynptr a unique id to uniquely associate slices to it. */
 2245	reg->id = dynptr_id;
 2246	reg->dynptr.type = type;
 2247	reg->dynptr.first_slot = first_slot;
 2248}
 2249
 2250static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
 2251{
 2252	if (base_type(reg->type) == PTR_TO_MAP_VALUE) {
 2253		const struct bpf_map *map = reg->map_ptr;
 2254
 2255		if (map->inner_map_meta) {
 2256			reg->type = CONST_PTR_TO_MAP;
 2257			reg->map_ptr = map->inner_map_meta;
 2258			/* transfer reg's id which is unique for every map_lookup_elem
 2259			 * as UID of the inner map.
 2260			 */
 2261			if (btf_record_has_field(map->inner_map_meta->record,
 2262						 BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) {
 2263				reg->map_uid = reg->id;
 2264			}
 2265		} else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
 2266			reg->type = PTR_TO_XDP_SOCK;
 2267		} else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
 2268			   map->map_type == BPF_MAP_TYPE_SOCKHASH) {
 2269			reg->type = PTR_TO_SOCKET;
 2270		} else {
 2271			reg->type = PTR_TO_MAP_VALUE;
 2272		}
 2273		return;
 2274	}
 2275
 2276	reg->type &= ~PTR_MAYBE_NULL;
 2277}
 2278
 2279static void mark_reg_graph_node(struct bpf_reg_state *regs, u32 regno,
 2280				struct btf_field_graph_root *ds_head)
 2281{
 2282	__mark_reg_known_zero(&regs[regno]);
 2283	regs[regno].type = PTR_TO_BTF_ID | MEM_ALLOC;
 2284	regs[regno].btf = ds_head->btf;
 2285	regs[regno].btf_id = ds_head->value_btf_id;
 2286	regs[regno].off = ds_head->node_offset;
 2287}
 2288
 2289static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
 2290{
 2291	return type_is_pkt_pointer(reg->type);
 2292}
 2293
 2294static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)
 2295{
 2296	return reg_is_pkt_pointer(reg) ||
 2297	       reg->type == PTR_TO_PACKET_END;
 2298}
 2299
 2300static bool reg_is_dynptr_slice_pkt(const struct bpf_reg_state *reg)
 2301{
 2302	return base_type(reg->type) == PTR_TO_MEM &&
 2303	       (reg->type &
 2304		(DYNPTR_TYPE_SKB | DYNPTR_TYPE_XDP | DYNPTR_TYPE_SKB_META));
 2305}
 2306
 2307/* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */
 2308static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,
 2309				    enum bpf_reg_type which)
 2310{
 2311	/* The register can already have a range from prior markings.
 2312	 * This is fine as long as it hasn't been advanced from its
 2313	 * origin.
 2314	 */
 2315	return reg->type == which &&
 2316	       reg->id == 0 &&
 2317	       reg->off == 0 &&
 2318	       tnum_equals_const(reg->var_off, 0);
 2319}
 2320
 2321/* Reset the min/max bounds of a register */
 2322static void __mark_reg_unbounded(struct bpf_reg_state *reg)
 2323{
 2324	reg->smin_value = S64_MIN;
 2325	reg->smax_value = S64_MAX;
 2326	reg->umin_value = 0;
 2327	reg->umax_value = U64_MAX;
 2328
 2329	reg->s32_min_value = S32_MIN;
 2330	reg->s32_max_value = S32_MAX;
 2331	reg->u32_min_value = 0;
 2332	reg->u32_max_value = U32_MAX;
 2333}
 2334
 2335static void __mark_reg64_unbounded(struct bpf_reg_state *reg)
 2336{
 2337	reg->smin_value = S64_MIN;
 2338	reg->smax_value = S64_MAX;
 2339	reg->umin_value = 0;
 2340	reg->umax_value = U64_MAX;
 2341}
 2342
 2343static void __mark_reg32_unbounded(struct bpf_reg_state *reg)
 2344{
 2345	reg->s32_min_value = S32_MIN;
 2346	reg->s32_max_value = S32_MAX;
 2347	reg->u32_min_value = 0;
 2348	reg->u32_max_value = U32_MAX;
 2349}
 2350
 2351static void __update_reg32_bounds(struct bpf_reg_state *reg)
 2352{
 2353	struct tnum var32_off = tnum_subreg(reg->var_off);
 2354
 2355	/* min signed is max(sign bit) | min(other bits) */
 2356	reg->s32_min_value = max_t(s32, reg->s32_min_value,
 2357			var32_off.value | (var32_off.mask & S32_MIN));
 2358	/* max signed is min(sign bit) | max(other bits) */
 2359	reg->s32_max_value = min_t(s32, reg->s32_max_value,
 2360			var32_off.value | (var32_off.mask & S32_MAX));
 2361	reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value);
 2362	reg->u32_max_value = min(reg->u32_max_value,
 2363				 (u32)(var32_off.value | var32_off.mask));
 2364}
 2365
 2366static void __update_reg64_bounds(struct bpf_reg_state *reg)
 2367{
 2368	u64 tnum_next, tmax;
 2369	bool umin_in_tnum;
 2370
 2371	/* min signed is max(sign bit) | min(other bits) */
 2372	reg->smin_value = max_t(s64, reg->smin_value,
 2373				reg->var_off.value | (reg->var_off.mask & S64_MIN));
 2374	/* max signed is min(sign bit) | max(other bits) */
 2375	reg->smax_value = min_t(s64, reg->smax_value,
 2376				reg->var_off.value | (reg->var_off.mask & S64_MAX));
 2377	reg->umin_value = max(reg->umin_value, reg->var_off.value);
 2378	reg->umax_value = min(reg->umax_value,
 2379			      reg->var_off.value | reg->var_off.mask);
 2380
 2381	/* Check if u64 and tnum overlap in a single value */
 2382	tnum_next = tnum_step(reg->var_off, reg->umin_value);
 2383	umin_in_tnum = (reg->umin_value & ~reg->var_off.mask) == reg->var_off.value;
 2384	tmax = reg->var_off.value | reg->var_off.mask;
 2385	if (umin_in_tnum && tnum_next > reg->umax_value) {
 2386		/* The u64 range and the tnum only overlap in umin.
 2387		 * u64:  ---[xxxxxx]-----
 2388		 * tnum: --xx----------x-
 2389		 */
 2390		___mark_reg_known(reg, reg->umin_value);
 2391	} else if (!umin_in_tnum && tnum_next == tmax) {
 2392		/* The u64 range and the tnum only overlap in the maximum value
 2393		 * represented by the tnum, called tmax.
 2394		 * u64:  ---[xxxxxx]-----
 2395		 * tnum: xx-----x--------
 2396		 */
 2397		___mark_reg_known(reg, tmax);
 2398	} else if (!umin_in_tnum && tnum_next <= reg->umax_value &&
 2399		   tnum_step(reg->var_off, tnum_next) > reg->umax_value) {
 2400		/* The u64 range and the tnum only overlap in between umin
 2401		 * (excluded) and umax.
 2402		 * u64:  ---[xxxxxx]-----
 2403		 * tnum: xx----x-------x-
 2404		 */
 2405		___mark_reg_known(reg, tnum_next);
 2406	}
 2407}
 2408
 2409static void __update_reg_bounds(struct bpf_reg_state *reg)
 2410{
 2411	__update_reg32_bounds(reg);
 2412	__update_reg64_bounds(reg);
 2413}
 2414
 2415/* Uses signed min/max values to inform unsigned, and vice-versa */
 2416static void __reg32_deduce_bounds(struct bpf_reg_state *reg)
 2417{
 2418	/* If upper 32 bits of u64/s64 range don't change, we can use lower 32
 2419	 * bits to improve our u32/s32 boundaries.
 2420	 *
 2421	 * E.g., the case where we have upper 32 bits as zero ([10, 20] in
 2422	 * u64) is pretty trivial, it's obvious that in u32 we'll also have
 2423	 * [10, 20] range. But this property holds for any 64-bit range as
 2424	 * long as upper 32 bits in that entire range of values stay the same.
 2425	 *
 2426	 * E.g., u64 range [0x10000000A, 0x10000000F] ([4294967306, 4294967311]
 2427	 * in decimal) has the same upper 32 bits throughout all the values in
 2428	 * that range. As such, lower 32 bits form a valid [0xA, 0xF] ([10, 15])
 2429	 * range.
 2430	 *
 2431	 * Note also, that [0xA, 0xF] is a valid range both in u32 and in s32,
 2432	 * following the rules outlined below about u64/s64 correspondence
 2433	 * (which equally applies to u32 vs s32 correspondence). In general it
 2434	 * depends on actual hexadecimal values of 32-bit range. They can form
 2435	 * only valid u32, or only valid s32 ranges in some cases.
 2436	 *
 2437	 * So we use all these insights to derive bounds for subregisters here.
 2438	 */
 2439	if ((reg->umin_value >> 32) == (reg->umax_value >> 32)) {
 2440		/* u64 to u32 casting preserves validity of low 32 bits as
 2441		 * a range, if upper 32 bits are the same
 2442		 */
 2443		reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)reg->umin_value);
 2444		reg->u32_max_value = min_t(u32, reg->u32_max_value, (u32)reg->umax_value);
 2445
 2446		if ((s32)reg->umin_value <= (s32)reg->umax_value) {
 2447			reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->umin_value);
 2448			reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->umax_value);
 2449		}
 2450	}
 2451	if ((reg->smin_value >> 32) == (reg->smax_value >> 32)) {
 2452		/* low 32 bits should form a proper u32 range */
 2453		if ((u32)reg->smin_value <= (u32)reg->smax_value) {
 2454			reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)reg->smin_value);
 2455			reg->u32_max_value = min_t(u32, reg->u32_max_value, (u32)reg->smax_value);
 2456		}
 2457		/* low 32 bits should form a proper s32 range */
 2458		if ((s32)reg->smin_value <= (s32)reg->smax_value) {
 2459			reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->smin_value);
 2460			reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->smax_value);
 2461		}
 2462	}
 2463	/* Special case where upper bits form a small sequence of two
 2464	 * sequential numbers (in 32-bit unsigned space, so 0xffffffff to
 2465	 * 0x00000000 is also valid), while lower bits form a proper s32 range
 2466	 * going from negative numbers to positive numbers. E.g., let's say we
 2467	 * have s64 range [-1, 1] ([0xffffffffffffffff, 0x0000000000000001]).
 2468	 * Possible s64 values are {-1, 0, 1} ({0xffffffffffffffff,
 2469	 * 0x0000000000000000, 0x00000000000001}). Ignoring upper 32 bits,
 2470	 * we still get a valid s32 range [-1, 1] ([0xffffffff, 0x00000001]).
 2471	 * Note that it doesn't have to be 0xffffffff going to 0x00000000 in
 2472	 * upper 32 bits. As a random example, s64 range
 2473	 * [0xfffffff0fffffff0; 0xfffffff100000010], forms a valid s32 range
 2474	 * [-16, 16] ([0xfffffff0; 0x00000010]) in its 32 bit subregister.
 2475	 */
 2476	if ((u32)(reg->umin_value >> 32) + 1 == (u32)(reg->umax_value >> 32) &&
 2477	    (s32)reg->umin_value < 0 && (s32)reg->umax_value >= 0) {
 2478		reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->umin_value);
 2479		reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->umax_value);
 2480	}
 2481	if ((u32)(reg->smin_value >> 32) + 1 == (u32)(reg->smax_value >> 32) &&
 2482	    (s32)reg->smin_value < 0 && (s32)reg->smax_value >= 0) {
 2483		reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->smin_value);
 2484		reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->smax_value);
 2485	}
 2486	/* if u32 range forms a valid s32 range (due to matching sign bit),
 2487	 * try to learn from that
 2488	 */
 2489	if ((s32)reg->u32_min_value <= (s32)reg->u32_max_value) {
 2490		reg->s32_min_value = max_t(s32, reg->s32_min_value, reg->u32_min_value);
 2491		reg->s32_max_value = min_t(s32, reg->s32_max_value, reg->u32_max_value);
 2492	}
 2493	/* If we cannot cross the sign boundary, then signed and unsigned bounds
 2494	 * are the same, so combine.  This works even in the negative case, e.g.
 2495	 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
 2496	 */
 2497	if ((u32)reg->s32_min_value <= (u32)reg->s32_max_value) {
 2498		reg->u32_min_value = max_t(u32, reg->s32_min_value, reg->u32_min_value);
 2499		reg->u32_max_value = min_t(u32, reg->s32_max_value, reg->u32_max_value);
 2500	} else {
 2501		if (reg->u32_max_value < (u32)reg->s32_min_value) {
 2502			/* See __reg64_deduce_bounds() for detailed explanation.
 2503			 * Refine ranges in the following situation:
 2504			 *
 2505			 * 0                                                   U32_MAX
 2506			 * |  [xxxxxxxxxxxxxx u32 range xxxxxxxxxxxxxx]              |
 2507			 * |----------------------------|----------------------------|
 2508			 * |xxxxx s32 range xxxxxxxxx]                       [xxxxxxx|
 2509			 * 0                     S32_MAX S32_MIN                    -1
 2510			 */
 2511			reg->s32_min_value = (s32)reg->u32_min_value;
 2512			reg->u32_max_value = min_t(u32, reg->u32_max_value, reg->s32_max_value);
 2513		} else if ((u32)reg->s32_max_value < reg->u32_min_value) {
 2514			/*
 2515			 * 0                                                   U32_MAX
 2516			 * |              [xxxxxxxxxxxxxx u32 range xxxxxxxxxxxxxx]  |
 2517			 * |----------------------------|----------------------------|
 2518			 * |xxxxxxxxx]                       [xxxxxxxxxxxx s32 range |
 2519			 * 0                     S32_MAX S32_MIN                    -1
 2520			 */
 2521			reg->s32_max_value = (s32)reg->u32_max_value;
 2522			reg->u32_min_value = max_t(u32, reg->u32_min_value, reg->s32_min_value);
 2523		}
 2524	}
 2525}
 2526
 2527static void __reg64_deduce_bounds(struct bpf_reg_state *reg)
 2528{
 2529	/* If u64 range forms a valid s64 range (due to matching sign bit),
 2530	 * try to learn from that. Let's do a bit of ASCII art to see when
 2531	 * this is happening. Let's take u64 range first:
 2532	 *
 2533	 * 0             0x7fffffffffffffff 0x8000000000000000        U64_MAX
 2534	 * |-------------------------------|--------------------------------|
 2535	 *
 2536	 * Valid u64 range is formed when umin and umax are anywhere in the
 2537	 * range [0, U64_MAX], and umin <= umax. u64 case is simple and
 2538	 * straightforward. Let's see how s64 range maps onto the same range
 2539	 * of values, annotated below the line for comparison:
 2540	 *
 2541	 * 0             0x7fffffffffffffff 0x8000000000000000        U64_MAX
 2542	 * |-------------------------------|--------------------------------|
 2543	 * 0                        S64_MAX S64_MIN                        -1
 2544	 *
 2545	 * So s64 values basically start in the middle and they are logically
 2546	 * contiguous to the right of it, wrapping around from -1 to 0, and
 2547	 * then finishing as S64_MAX (0x7fffffffffffffff) right before
 2548	 * S64_MIN. We can try drawing the continuity of u64 vs s64 values
 2549	 * more visually as mapped to sign-agnostic range of hex values.
 2550	 *
 2551	 *  u64 start                                               u64 end
 2552	 *  _______________________________________________________________
 2553	 * /                                                               \
 2554	 * 0             0x7fffffffffffffff 0x8000000000000000        U64_MAX
 2555	 * |-------------------------------|--------------------------------|
 2556	 * 0                        S64_MAX S64_MIN                        -1
 2557	 *                                / \
 2558	 * >------------------------------   ------------------------------->
 2559	 * s64 continues...        s64 end   s64 start          s64 "midpoint"
 2560	 *
 2561	 * What this means is that, in general, we can't always derive
 2562	 * something new about u64 from any random s64 range, and vice versa.
 2563	 *
 2564	 * But we can do that in two particular cases. One is when entire
 2565	 * u64/s64 range is *entirely* contained within left half of the above
 2566	 * diagram or when it is *entirely* contained in the right half. I.e.:
 2567	 *
 2568	 * |-------------------------------|--------------------------------|
 2569	 *     ^                   ^            ^                 ^
 2570	 *     A                   B            C                 D
 2571	 *
 2572	 * [A, B] and [C, D] are contained entirely in their respective halves
 2573	 * and form valid contiguous ranges as both u64 and s64 values. [A, B]
 2574	 * will be non-negative both as u64 and s64 (and in fact it will be
 2575	 * identical ranges no matter the signedness). [C, D] treated as s64
 2576	 * will be a range of negative values, while in u64 it will be
 2577	 * non-negative range of values larger than 0x8000000000000000.
 2578	 *
 2579	 * Now, any other range here can't be represented in both u64 and s64
 2580	 * simultaneously. E.g., [A, C], [A, D], [B, C], [B, D] are valid
 2581	 * contiguous u64 ranges, but they are discontinuous in s64. [B, C]
 2582	 * in s64 would be properly presented as [S64_MIN, C] and [B, S64_MAX],
 2583	 * for example. Similarly, valid s64 range [D, A] (going from negative
 2584	 * to positive values), would be two separate [D, U64_MAX] and [0, A]
 2585	 * ranges as u64. Currently reg_state can't represent two segments per
 2586	 * numeric domain, so in such situations we can only derive maximal
 2587	 * possible range ([0, U64_MAX] for u64, and [S64_MIN, S64_MAX] for s64).
 2588	 *
 2589	 * So we use these facts to derive umin/umax from smin/smax and vice
 2590	 * versa only if they stay within the same "half". This is equivalent
 2591	 * to checking sign bit: lower half will have sign bit as zero, upper
 2592	 * half have sign bit 1. Below in code we simplify this by just
 2593	 * casting umin/umax as smin/smax and checking if they form valid
 2594	 * range, and vice versa. Those are equivalent checks.
 2595	 */
 2596	if ((s64)reg->umin_value <= (s64)reg->umax_value) {
 2597		reg->smin_value = max_t(s64, reg->smin_value, reg->umin_value);
 2598		reg->smax_value = min_t(s64, reg->smax_value, reg->umax_value);
 2599	}
 2600	/* If we cannot cross the sign boundary, then signed and unsigned bounds
 2601	 * are the same, so combine.  This works even in the negative case, e.g.
 2602	 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
 2603	 */
 2604	if ((u64)reg->smin_value <= (u64)reg->smax_value) {
 2605		reg->umin_value = max_t(u64, reg->smin_value, reg->umin_value);
 2606		reg->umax_value = min_t(u64, reg->smax_value, reg->umax_value);
 2607	} else {
 2608		/* If the s64 range crosses the sign boundary, then it's split
 2609		 * between the beginning and end of the U64 domain. In that
 2610		 * case, we can derive new bounds if the u64 range overlaps
 2611		 * with only one end of the s64 range.
 2612		 *
 2613		 * In the following example, the u64 range overlaps only with
 2614		 * positive portion of the s64 range.
 2615		 *
 2616		 * 0                                                   U64_MAX
 2617		 * |  [xxxxxxxxxxxxxx u64 range xxxxxxxxxxxxxx]              |
 2618		 * |----------------------------|----------------------------|
 2619		 * |xxxxx s64 range xxxxxxxxx]                       [xxxxxxx|
 2620		 * 0                     S64_MAX S64_MIN                    -1
 2621		 *
 2622		 * We can thus derive the following new s64 and u64 ranges.
 2623		 *
 2624		 * 0                                                   U64_MAX
 2625		 * |  [xxxxxx u64 range xxxxx]                               |
 2626		 * |----------------------------|----------------------------|
 2627		 * |  [xxxxxx s64 range xxxxx]                               |
 2628		 * 0                     S64_MAX S64_MIN                    -1
 2629		 *
 2630		 * If they overlap in two places, we can't derive anything
 2631		 * because reg_state can't represent two ranges per numeric
 2632		 * domain.
 2633		 *
 2634		 * 0                                                   U64_MAX
 2635		 * |  [xxxxxxxxxxxxxxxxx u64 range xxxxxxxxxxxxxxxxx]        |
 2636		 * |----------------------------|----------------------------|
 2637		 * |xxxxx s64 range xxxxxxxxx]                    [xxxxxxxxxx|
 2638		 * 0                     S64_MAX S64_MIN                    -1
 2639		 *
 2640		 * The first condition below corresponds to the first diagram
 2641		 * above.
 2642		 */
 2643		if (reg->umax_value < (u64)reg->smin_value) {
 2644			reg->smin_value = (s64)reg->umin_value;
 2645			reg->umax_value = min_t(u64, reg->umax_value, reg->smax_value);
 2646		} else if ((u64)reg->smax_value < reg->umin_value) {
 2647			/* This second condition considers the case where the u64 range
 2648			 * overlaps with the negative portion of the s64 range:
 2649			 *
 2650			 * 0                                                   U64_MAX
 2651			 * |              [xxxxxxxxxxxxxx u64 range xxxxxxxxxxxxxx]  |
 2652			 * |----------------------------|----------------------------|
 2653			 * |xxxxxxxxx]                       [xxxxxxxxxxxx s64 range |
 2654			 * 0                     S64_MAX S64_MIN                    -1
 2655			 */
 2656			reg->smax_value = (s64)reg->umax_value;
 2657			reg->umin_value = max_t(u64, reg->umin_value, reg->smin_value);
 2658		}
 2659	}
 2660}
 2661
 2662static void __reg_deduce_mixed_bounds(struct bpf_reg_state *reg)
 2663{
 2664	/* Try to tighten 64-bit bounds from 32-bit knowledge, using 32-bit
 2665	 * values on both sides of 64-bit range in hope to have tighter range.
 2666	 * E.g., if r1 is [0x1'00000000, 0x3'80000000], and we learn from
 2667	 * 32-bit signed > 0 operation that s32 bounds are now [1; 0x7fffffff].
 2668	 * With this, we can substitute 1 as low 32-bits of _low_ 64-bit bound
 2669	 * (0x100000000 -> 0x100000001) and 0x7fffffff as low 32-bits of
 2670	 * _high_ 64-bit bound (0x380000000 -> 0x37fffffff) and arrive at a
 2671	 * better overall bounds for r1 as [0x1'000000001; 0x3'7fffffff].
 2672	 * We just need to make sure that derived bounds we are intersecting
 2673	 * with are well-formed ranges in respective s64 or u64 domain, just
 2674	 * like we do with similar kinds of 32-to-64 or 64-to-32 adjustments.
 2675	 */
 2676	__u64 new_umin, new_umax;
 2677	__s64 new_smin, new_smax;
 2678
 2679	/* u32 -> u64 tightening, it's always well-formed */
 2680	new_umin = (reg->umin_value & ~0xffffffffULL) | reg->u32_min_value;
 2681	new_umax = (reg->umax_value & ~0xffffffffULL) | reg->u32_max_value;
 2682	reg->umin_value = max_t(u64, reg->umin_value, new_umin);
 2683	reg->umax_value = min_t(u64, reg->umax_value, new_umax);
 2684	/* u32 -> s64 tightening, u32 range embedded into s64 preserves range validity */
 2685	new_smin = (reg->smin_value & ~0xffffffffULL) | reg->u32_min_value;
 2686	new_smax = (reg->smax_value & ~0xffffffffULL) | reg->u32_max_value;
 2687	reg->smin_value = max_t(s64, reg->smin_value, new_smin);
 2688	reg->smax_value = min_t(s64, reg->smax_value, new_smax);
 2689
 2690	/* Here we would like to handle a special case after sign extending load,
 2691	 * when upper bits for a 64-bit range are all 1s or all 0s.
 2692	 *
 2693	 * Upper bits are all 1s when register is in a range:
 2694	 *   [0xffff_ffff_0000_0000, 0xffff_ffff_ffff_ffff]
 2695	 * Upper bits are all 0s when register is in a range:
 2696	 *   [0x0000_0000_0000_0000, 0x0000_0000_ffff_ffff]
 2697	 * Together this forms are continuous range:
 2698	 *   [0xffff_ffff_0000_0000, 0x0000_0000_ffff_ffff]
 2699	 *
 2700	 * Now, suppose that register range is in fact tighter:
 2701	 *   [0xffff_ffff_8000_0000, 0x0000_0000_ffff_ffff] (R)
 2702	 * Also suppose that it's 32-bit range is positive,
 2703	 * meaning that lower 32-bits of the full 64-bit register
 2704	 * are in the range:
 2705	 *   [0x0000_0000, 0x7fff_ffff] (W)
 2706	 *
 2707	 * If this happens, then any value in a range:
 2708	 *   [0xffff_ffff_0000_0000, 0xffff_ffff_7fff_ffff]
 2709	 * is smaller than a lowest bound of the range (R):
 2710	 *   0xffff_ffff_8000_0000
 2711	 * which means that upper bits of the full 64-bit register
 2712	 * can't be all 1s, when lower bits are in range (W).
 2713	 *
 2714	 * Note that:
 2715	 *  - 0xffff_ffff_8000_0000 == (s64)S32_MIN
 2716	 *  - 0x0000_0000_7fff_ffff == (s64)S32_MAX
 2717	 * These relations are used in the conditions below.
 2718	 */
 2719	if (reg->s32_min_value >= 0 && reg->smin_value >= S32_MIN && reg->smax_value <= S32_MAX) {
 2720		reg->smin_value = reg->s32_min_value;
 2721		reg->smax_value = reg->s32_max_value;
 2722		reg->umin_value = reg->s32_min_value;
 2723		reg->umax_value = reg->s32_max_value;
 2724		reg->var_off = tnum_intersect(reg->var_off,
 2725					      tnum_range(reg->smin_value, reg->smax_value));
 2726	}
 2727}
 2728
 2729static void __reg_deduce_bounds(struct bpf_reg_state *reg)
 2730{
 2731	__reg32_deduce_bounds(reg);
 2732	__reg64_deduce_bounds(reg);
 2733	__reg_deduce_mixed_bounds(reg);
 2734}
 2735
 2736/* Attempts to improve var_off based on unsigned min/max information */
 2737static void __reg_bound_offset(struct bpf_reg_state *reg)
 2738{
 2739	struct tnum var64_off = tnum_intersect(reg->var_off,
 2740					       tnum_range(reg->umin_value,
 2741							  reg->umax_value));
 2742	struct tnum var32_off = tnum_intersect(tnum_subreg(var64_off),
 2743					       tnum_range(reg->u32_min_value,
 2744							  reg->u32_max_value));
 2745
 2746	reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off);
 2747}
 2748
 2749static void reg_bounds_sync(struct bpf_reg_state *reg)
 2750{
 2751	/* We might have learned new bounds from the var_off. */
 2752	__update_reg_bounds(reg);
 2753	/* We might have learned something about the sign bit. */
 2754	__reg_deduce_bounds(reg);
 2755	__reg_deduce_bounds(reg);
 2756	__reg_deduce_bounds(reg);
 2757	/* We might have learned some bits from the bounds. */
 2758	__reg_bound_offset(reg);
 2759	/* Intersecting with the old var_off might have improved our bounds
 2760	 * slightly, e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
 2761	 * then new var_off is (0; 0x7f...fc) which improves our umax.
 2762	 */
 2763	__update_reg_bounds(reg);
 2764}
 2765
 2766static int reg_bounds_sanity_check(struct bpf_verifier_env *env,
 2767				   struct bpf_reg_state *reg, const char *ctx)
 2768{
 2769	const char *msg;
 2770
 2771	if (reg->umin_value > reg->umax_value ||
 2772	    reg->smin_value > reg->smax_value ||
 2773	    reg->u32_min_value > reg->u32_max_value ||
 2774	    reg->s32_min_value > reg->s32_max_value) {
 2775		    msg = "range bounds violation";
 2776		    goto out;
 2777	}
 2778
 2779	if (tnum_is_const(reg->var_off)) {
 2780		u64 uval = reg->var_off.value;
 2781		s64 sval = (s64)uval;
 2782
 2783		if (reg->umin_value != uval || reg->umax_value != uval ||
 2784		    reg->smin_value != sval || reg->smax_value != sval) {
 2785			msg = "const tnum out of sync with range bounds";
 2786			goto out;
 2787		}
 2788	}
 2789
 2790	if (tnum_subreg_is_const(reg->var_off)) {
 2791		u32 uval32 = tnum_subreg(reg->var_off).value;
 2792		s32 sval32 = (s32)uval32;
 2793
 2794		if (reg->u32_min_value != uval32 || reg->u32_max_value != uval32 ||
 2795		    reg->s32_min_value != sval32 || reg->s32_max_value != sval32) {
 2796			msg = "const subreg tnum out of sync with range bounds";
 2797			goto out;
 2798		}
 2799	}
 2800
 2801	return 0;
 2802out:
 2803	verifier_bug(env, "REG INVARIANTS VIOLATION (%s): %s u64=[%#llx, %#llx] "
 2804		     "s64=[%#llx, %#llx] u32=[%#x, %#x] s32=[%#x, %#x] var_off=(%#llx, %#llx)",
 2805		     ctx, msg, reg->umin_value, reg->umax_value,
 2806		     reg->smin_value, reg->smax_value,
 2807		     reg->u32_min_value, reg->u32_max_value,
 2808		     reg->s32_min_value, reg->s32_max_value,
 2809		     reg->var_off.value, reg->var_off.mask);
 2810	if (env->test_reg_invariants)
 2811		return -EFAULT;
 2812	__mark_reg_unbounded(reg);
 2813	return 0;
 2814}
 2815
 2816static bool __reg32_bound_s64(s32 a)
 2817{
 2818	return a >= 0 && a <= S32_MAX;
 2819}
 2820
 2821static void __reg_assign_32_into_64(struct bpf_reg_state *reg)
 2822{
 2823	reg->umin_value = reg->u32_min_value;
 2824	reg->umax_value = reg->u32_max_value;
 2825
 2826	/* Attempt to pull 32-bit signed bounds into 64-bit bounds but must
 2827	 * be positive otherwise set to worse case bounds and refine later
 2828	 * from tnum.
 2829	 */
 2830	if (__reg32_bound_s64(reg->s32_min_value) &&
 2831	    __reg32_bound_s64(reg->s32_max_value)) {
 2832		reg->smin_value = reg->s32_min_value;
 2833		reg->smax_value = reg->s32_max_value;
 2834	} else {
 2835		reg->smin_value = 0;
 2836		reg->smax_value = U32_MAX;
 2837	}
 2838}
 2839
 2840/* Mark a register as having a completely unknown (scalar) value. */
 2841static void __mark_reg_unknown_imprecise(struct bpf_reg_state *reg)
 2842{
 2843	/*
 2844	 * Clear type, off, and union(map_ptr, range) and
 2845	 * padding between 'type' and union
 2846	 */
 2847	memset(reg, 0, offsetof(struct bpf_reg_state, var_off));
 2848	reg->type = SCALAR_VALUE;
 2849	reg->id = 0;
 2850	reg->ref_obj_id = 0;
 2851	reg->var_off = tnum_unknown;
 2852	reg->frameno = 0;
 2853	reg->precise = false;
 2854	__mark_reg_unbounded(reg);
 2855}
 2856
 2857/* Mark a register as having a completely unknown (scalar) value,
 2858 * initialize .precise as true when not bpf capable.
 2859 */
 2860static void __mark_reg_unknown(const struct bpf_verifier_env *env,
 2861			       struct bpf_reg_state *reg)
 2862{
 2863	__mark_reg_unknown_imprecise(reg);
 2864	reg->precise = !env->bpf_capable;
 2865}
 2866
 2867static void mark_reg_unknown(struct bpf_verifier_env *env,
 2868			     struct bpf_reg_state *regs, u32 regno)
 2869{
 2870	if (WARN_ON(regno >= MAX_BPF_REG)) {
 2871		verbose(env, "mark_reg_unknown(regs, %u)\n", regno);
 2872		/* Something bad happened, let's kill all regs except FP */
 2873		for (regno = 0; regno < BPF_REG_FP; regno++)
 2874			__mark_reg_not_init(env, regs + regno);
 2875		return;
 2876	}
 2877	__mark_reg_unknown(env, regs + regno);
 2878}
 2879
 2880static int __mark_reg_s32_range(struct bpf_verifier_env *env,
 2881				struct bpf_reg_state *regs,
 2882				u32 regno,
 2883				s32 s32_min,
 2884				s32 s32_max)
 2885{
 2886	struct bpf_reg_state *reg = regs + regno;
 2887
 2888	reg->s32_min_value = max_t(s32, reg->s32_min_value, s32_min);
 2889	reg->s32_max_value = min_t(s32, reg->s32_max_value, s32_max);
 2890
 2891	reg->smin_value = max_t(s64, reg->smin_value, s32_min);
 2892	reg->smax_value = min_t(s64, reg->smax_value, s32_max);
 2893
 2894	reg_bounds_sync(reg);
 2895
 2896	return reg_bounds_sanity_check(env, reg, "s32_range");
 2897}
 2898
 2899static void __mark_reg_not_init(const struct bpf_verifier_env *env,
 2900				struct bpf_reg_state *reg)
 2901{
 2902	__mark_reg_unknown(env, reg);
 2903	reg->type = NOT_INIT;
 2904}
 2905
 2906static void mark_reg_not_init(struct bpf_verifier_env *env,
 2907			      struct bpf_reg_state *regs, u32 regno)
 2908{
 2909	if (WARN_ON(regno >= MAX_BPF_REG)) {
 2910		verbose(env, "mark_reg_not_init(regs, %u)\n", regno);
 2911		/* Something bad happened, let's kill all regs except FP */
 2912		for (regno = 0; regno < BPF_REG_FP; regno++)
 2913			__mark_reg_not_init(env, regs + regno);
 2914		return;
 2915	}
 2916	__mark_reg_not_init(env, regs + regno);
 2917}
 2918
 2919static int mark_btf_ld_reg(struct bpf_verifier_env *env,
 2920			   struct bpf_reg_state *regs, u32 regno,
 2921			   enum bpf_reg_type reg_type,
 2922			   struct btf *btf, u32 btf_id,
 2923			   enum bpf_type_flag flag)
 2924{
 2925	switch (reg_type) {
 2926	case SCALAR_VALUE:
 2927		mark_reg_unknown(env, regs, regno);
 2928		return 0;
 2929	case PTR_TO_BTF_ID:
 2930		mark_reg_known_zero(env, regs, regno);
 2931		regs[regno].type = PTR_TO_BTF_ID | flag;
 2932		regs[regno].btf = btf;
 2933		regs[regno].btf_id = btf_id;
 2934		if (type_may_be_null(flag))
 2935			regs[regno].id = ++env->id_gen;
 2936		return 0;
 2937	case PTR_TO_MEM:
 2938		mark_reg_known_zero(env, regs, regno);
 2939		regs[regno].type = PTR_TO_MEM | flag;
 2940		regs[regno].mem_size = 0;
 2941		return 0;
 2942	default:
 2943		verifier_bug(env, "unexpected reg_type %d in %s\n", reg_type, __func__);
 2944		return -EFAULT;
 2945	}
 2946}
 2947
 2948#define DEF_NOT_SUBREG	(0)
 2949static void init_reg_state(struct bpf_verifier_env *env,
 2950			   struct bpf_func_state *state)
 2951{
 2952	struct bpf_reg_state *regs = state->regs;
 2953	int i;
 2954
 2955	for (i = 0; i < MAX_BPF_REG; i++) {
 2956		mark_reg_not_init(env, regs, i);
 2957		regs[i].subreg_def = DEF_NOT_SUBREG;
 2958	}
 2959
 2960	/* frame pointer */
 2961	regs[BPF_REG_FP].type = PTR_TO_STACK;
 2962	mark_reg_known_zero(env, regs, BPF_REG_FP);
 2963	regs[BPF_REG_FP].frameno = state->frameno;
 2964}
 2965
 2966static struct bpf_retval_range retval_range(s32 minval, s32 maxval)
 2967{
 2968	return (struct bpf_retval_range){ minval, maxval };
 2969}
 2970
 2971#define BPF_MAIN_FUNC (-1)
 2972static void init_func_state(struct bpf_verifier_env *env,
 2973			    struct bpf_func_state *state,
 2974			    int callsite, int frameno, int subprogno)
 2975{
 2976	state->callsite = callsite;
 2977	state->frameno = frameno;
 2978	state->subprogno = subprogno;
 2979	state->callback_ret_range = retval_range(0, 0);
 2980	init_reg_state(env, state);
 2981	mark_verifier_state_scratched(env);
 2982}
 2983
 2984/* Similar to push_stack(), but for async callbacks */
 2985static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
 2986						int insn_idx, int prev_insn_idx,
 2987						int subprog, bool is_sleepable)
 2988{
 2989	struct bpf_verifier_stack_elem *elem;
 2990	struct bpf_func_state *frame;
 2991
 2992	elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL_ACCOUNT);
 2993	if (!elem)
 2994		return ERR_PTR(-ENOMEM);
 2995
 2996	elem->insn_idx = insn_idx;
 2997	elem->prev_insn_idx = prev_insn_idx;
 2998	elem->next = env->head;
 2999	elem->log_pos = env->log.end_pos;
 3000	env->head = elem;
 3001	env->stack_size++;
 3002	if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
 3003		verbose(env,
 3004			"The sequence of %d jumps is too complex for async cb.\n",
 3005			env->stack_size);
 3006		return ERR_PTR(-E2BIG);
 3007	}
 3008	/* Unlike push_stack() do not copy_verifier_state().
 3009	 * The caller state doesn't matter.
 3010	 * This is async callback. It starts in a fresh stack.
 3011	 * Initialize it similar to do_check_common().
 3012	 */
 3013	elem->st.branches = 1;
 3014	elem->st.in_sleepable = is_sleepable;
 3015	frame = kzalloc(sizeof(*frame), GFP_KERNEL_ACCOUNT);
 3016	if (!frame)
 3017		return ERR_PTR(-ENOMEM);
 3018	init_func_state(env, frame,
 3019			BPF_MAIN_FUNC /* callsite */,
 3020			0 /* frameno within this callchain */,
 3021			subprog /* subprog number within this prog */);
 3022	elem->st.frame[0] = frame;
 3023	return &elem->st;
 3024}
 3025
 3026
 3027enum reg_arg_type {
 3028	SRC_OP,		/* register is used as source operand */
 3029	DST_OP,		/* register is used as destination operand */
 3030	DST_OP_NO_MARK	/* same as above, check only, don't mark */
 3031};
 3032
 3033static int cmp_subprogs(const void *a, const void *b)
 3034{
 3035	return ((struct bpf_subprog_info *)a)->start -
 3036	       ((struct bpf_subprog_info *)b)->start;
 3037}
 3038
 3039/* Find subprogram that contains instruction at 'off' */
 3040struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off)
 3041{
 3042	struct bpf_subprog_info *vals = env->subprog_info;
 3043	int l, r, m;
 3044
 3045	if (off >= env->prog->len || off < 0 || env->subprog_cnt == 0)
 3046		return NULL;
 3047
 3048	l = 0;
 3049	r = env->subprog_cnt - 1;
 3050	while (l < r) {
 3051		m = l + (r - l + 1) / 2;
 3052		if (vals[m].start <= off)
 3053			l = m;
 3054		else
 3055			r = m - 1;
 3056	}
 3057	return &vals[l];
 3058}
 3059
 3060/* Find subprogram that starts exactly at 'off' */
 3061static int find_subprog(struct bpf_verifier_env *env, int off)
 3062{
 3063	struct bpf_subprog_info *p;
 3064
 3065	p = bpf_find_containing_subprog(env, off);
 3066	if (!p || p->start != off)
 3067		return -ENOENT;
 3068	return p - env->subprog_info;
 3069}
 3070
 3071static int add_subprog(struct bpf_verifier_env *env, int off)
 3072{
 3073	int insn_cnt = env->prog->len;
 3074	int ret;
 3075
 3076	if (off >= insn_cnt || off < 0) {
 3077		verbose(env, "call to invalid destination\n");
 3078		return -EINVAL;
 3079	}
 3080	ret = find_subprog(env, off);
 3081	if (ret >= 0)
 3082		return ret;
 3083	if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {
 3084		verbose(env, "too many subprograms\n");
 3085		return -E2BIG;
 3086	}
 3087	/* determine subprog starts. The end is one before the next starts */
 3088	env->subprog_info[env->subprog_cnt++].start = off;
 3089	sort(env->subprog_info, env->subprog_cnt,
 3090	     sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
 3091	return env->subprog_cnt - 1;
 3092}
 3093
 3094static int bpf_find_exception_callback_insn_off(struct bpf_verifier_env *env)
 3095{
 3096	struct bpf_prog_aux *aux = env->prog->aux;
 3097	struct btf *btf = aux->btf;
 3098	const struct btf_type *t;
 3099	u32 main_btf_id, id;
 3100	const char *name;
 3101	int ret, i;
 3102
 3103	/* Non-zero func_info_cnt implies valid btf */
 3104	if (!aux->func_info_cnt)
 3105		return 0;
 3106	main_btf_id = aux->func_info[0].type_id;
 3107
 3108	t = btf_type_by_id(btf, main_btf_id);
 3109	if (!t) {
 3110		verbose(env, "invalid btf id for main subprog in func_info\n");
 3111		return -EINVAL;
 3112	}
 3113
 3114	name = btf_find_decl_tag_value(btf, t, -1, "exception_callback:");
 3115	if (IS_ERR(name)) {
 3116		ret = PTR_ERR(name);
 3117		/* If there is no tag present, there is no exception callback */
 3118		if (ret == -ENOENT)
 3119			ret = 0;
 3120		else if (ret == -EEXIST)
 3121			verbose(env, "multiple exception callback tags for main subprog\n");
 3122		return ret;
 3123	}
 3124
 3125	ret = btf_find_by_name_kind(btf, name, BTF_KIND_FUNC);
 3126	if (ret < 0) {
 3127		verbose(env, "exception callback '%s' could not be found in BTF\n", name);
 3128		return ret;
 3129	}
 3130	id = ret;
 3131	t = btf_type_by_id(btf, id);
 3132	if (btf_func_linkage(t) != BTF_FUNC_GLOBAL) {
 3133		verbose(env, "exception callback '%s' must have global linkage\n", name);
 3134		return -EINVAL;
 3135	}
 3136	ret = 0;
 3137	for (i = 0; i < aux->func_info_cnt; i++) {
 3138		if (aux->func_info[i].type_id != id)
 3139			continue;
 3140		ret = aux->func_info[i].insn_off;
 3141		/* Further func_info and subprog checks will also happen
 3142		 * later, so assume this is the right insn_off for now.
 3143		 */
 3144		if (!ret) {
 3145			verbose(env, "invalid exception callback insn_off in func_info: 0\n");
 3146			ret = -EINVAL;
 3147		}
 3148	}
 3149	if (!ret) {
 3150		verbose(env, "exception callback type id not found in func_info\n");
 3151		ret = -EINVAL;
 3152	}
 3153	return ret;
 3154}
 3155
 3156#define MAX_KFUNC_DESCS 256
 3157#define MAX_KFUNC_BTFS	256
 3158
 3159struct bpf_kfunc_desc {
 3160	struct btf_func_model func_model;
 3161	u32 func_id;
 3162	s32 imm;
 3163	u16 offset;
 3164	unsigned long addr;
 3165};
 3166
 3167struct bpf_kfunc_btf {
 3168	struct btf *btf;
 3169	struct module *module;
 3170	u16 offset;
 3171};
 3172
 3173struct bpf_kfunc_desc_tab {
 3174	/* Sorted by func_id (BTF ID) and offset (fd_array offset) during
 3175	 * verification. JITs do lookups by bpf_insn, where func_id may not be
 3176	 * available, therefore at the end of verification do_misc_fixups()
 3177	 * sorts this by imm and offset.
 3178	 */
 3179	struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];
 3180	u32 nr_descs;
 3181};
 3182
 3183struct bpf_kfunc_btf_tab {
 3184	struct bpf_kfunc_btf descs[MAX_KFUNC_BTFS];
 3185	u32 nr_descs;
 3186};
 3187
 3188static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc,
 3189			    int insn_idx);
 3190
 3191static int kfunc_desc_cmp_by_id_off(const void *a, const void *b)
 3192{
 3193	const struct bpf_kfunc_desc *d0 = a;
 3194	const struct bpf_kfunc_desc *d1 = b;
 3195
 3196	/* func_id is not greater than BTF_MAX_TYPE */
 3197	return d0->func_id - d1->func_id ?: d0->offset - d1->offset;
 3198}
 3199
 3200static int kfunc_btf_cmp_by_off(const void *a, const void *b)
 3201{
 3202	const struct bpf_kfunc_btf *d0 = a;
 3203	const struct bpf_kfunc_btf *d1 = b;
 3204
 3205	return d0->offset - d1->offset;
 3206}
 3207
 3208static struct bpf_kfunc_desc *
 3209find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset)
 3210{
 3211	struct bpf_kfunc_desc desc = {
 3212		.func_id = func_id,
 3213		.offset = offset,
 3214	};
 3215	struct bpf_kfunc_desc_tab *tab;
 3216
 3217	tab = prog->aux->kfunc_tab;
 3218	return bsearch(&desc, tab->descs, tab->nr_descs,
 3219		       sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off);
 3220}
 3221
 3222int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
 3223		       u16 btf_fd_idx, u8 **func_addr)
 3224{
 3225	const struct bpf_kfunc_desc *desc;
 3226
 3227	desc = find_kfunc_desc(prog, func_id, btf_fd_idx);
 3228	if (!desc)
 3229		return -EFAULT;
 3230
 3231	*func_addr = (u8 *)desc->addr;
 3232	return 0;
 3233}
 3234
 3235static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,
 3236					 s16 offset)
 3237{
 3238	struct bpf_kfunc_btf kf_btf = { .offset = offset };
 3239	struct bpf_kfunc_btf_tab *tab;
 3240	struct bpf_kfunc_btf *b;
 3241	struct module *mod;
 3242	struct btf *btf;
 3243	int btf_fd;
 3244
 3245	tab = env->prog->aux->kfunc_btf_tab;
 3246	b = bsearch(&kf_btf, tab->descs, tab->nr_descs,
 3247		    sizeof(tab->descs[0]), kfunc_btf_cmp_by_off);
 3248	if (!b) {
 3249		if (tab->nr_descs == MAX_KFUNC_BTFS) {
 3250			verbose(env, "too many different module BTFs\n");
 3251			return ERR_PTR(-E2BIG);
 3252		}
 3253
 3254		if (bpfptr_is_null(env->fd_array)) {
 3255			verbose(env, "kfunc offset > 0 without fd_array is invalid\n");
 3256			return ERR_PTR(-EPROTO);
 3257		}
 3258
 3259		if (copy_from_bpfptr_offset(&btf_fd, env->fd_array,
 3260					    offset * sizeof(btf_fd),
 3261					    sizeof(btf_fd)))
 3262			return ERR_PTR(-EFAULT);
 3263
 3264		btf = btf_get_by_fd(btf_fd);
 3265		if (IS_ERR(btf)) {
 3266			verbose(env, "invalid module BTF fd specified\n");
 3267			return btf;
 3268		}
 3269
 3270		if (!btf_is_module(btf)) {
 3271			verbose(env, "BTF fd for kfunc is not a module BTF\n");
 3272			btf_put(btf);
 3273			return ERR_PTR(-EINVAL);
 3274		}
 3275
 3276		mod = btf_try_get_module(btf);
 3277		if (!mod) {
 3278			btf_put(btf);
 3279			return ERR_PTR(-ENXIO);
 3280		}
 3281
 3282		b = &tab->descs[tab->nr_descs++];
 3283		b->btf = btf;
 3284		b->module = mod;
 3285		b->offset = offset;
 3286
 3287		/* sort() reorders entries by value, so b may no longer point
 3288		 * to the right entry after this
 3289		 */
 3290		sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
 3291		     kfunc_btf_cmp_by_off, NULL);
 3292	} else {
 3293		btf = b->btf;
 3294	}
 3295
 3296	return btf;
 3297}
 3298
 3299void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab)
 3300{
 3301	if (!tab)
 3302		return;
 3303
 3304	while (tab->nr_descs--) {
 3305		module_put(tab->descs[tab->nr_descs].module);
 3306		btf_put(tab->descs[tab->nr_descs].btf);
 3307	}
 3308	kfree(tab);
 3309}
 3310
 3311static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset)
 3312{
 3313	if (offset) {
 3314		if (offset < 0) {
 3315			/* In the future, this can be allowed to increase limit
 3316			 * of fd index into fd_array, interpreted as u16.
 3317			 */
 3318			verbose(env, "negative offset disallowed for kernel module function call\n");
 3319			return ERR_PTR(-EINVAL);
 3320		}
 3321
 3322		return __find_kfunc_desc_btf(env, offset);
 3323	}
 3324	return btf_vmlinux ?: ERR_PTR(-ENOENT);
 3325}
 3326
 3327static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
 3328{
 3329	const struct btf_type *func, *func_proto;
 3330	struct bpf_kfunc_btf_tab *btf_tab;
 3331	struct btf_func_model func_model;
 3332	struct bpf_kfunc_desc_tab *tab;
 3333	struct bpf_prog_aux *prog_aux;
 3334	struct bpf_kfunc_desc *desc;
 3335	const char *func_name;
 3336	struct btf *desc_btf;
 3337	unsigned long addr;
 3338	int err;
 3339
 3340	prog_aux = env->prog->aux;
 3341	tab = prog_aux->kfunc_tab;
 3342	btf_tab = prog_aux->kfunc_btf_tab;
 3343	if (!tab) {
 3344		if (!btf_vmlinux) {
 3345			verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");
 3346			return -ENOTSUPP;
 3347		}
 3348
 3349		if (!env->prog->jit_requested) {
 3350			verbose(env, "JIT is required for calling kernel function\n");
 3351			return -ENOTSUPP;
 3352		}
 3353
 3354		if (!bpf_jit_supports_kfunc_call()) {
 3355			verbose(env, "JIT does not support calling kernel function\n");
 3356			return -ENOTSUPP;
 3357		}
 3358
 3359		if (!env->prog->gpl_compatible) {
 3360			verbose(env, "cannot call kernel function from non-GPL compatible program\n");
 3361			return -EINVAL;
 3362		}
 3363
 3364		tab = kzalloc(sizeof(*tab), GFP_KERNEL_ACCOUNT);
 3365		if (!tab)
 3366			return -ENOMEM;
 3367		prog_aux->kfunc_tab = tab;
 3368	}
 3369
 3370	/* func_id == 0 is always invalid, but instead of returning an error, be
 3371	 * conservative and wait until the code elimination pass before returning
 3372	 * error, so that invalid calls that get pruned out can be in BPF programs
 3373	 * loaded from userspace.  It is also required that offset be untouched
 3374	 * for such calls.
 3375	 */
 3376	if (!func_id && !offset)
 3377		return 0;
 3378
 3379	if (!btf_tab && offset) {
 3380		btf_tab = kzalloc(sizeof(*btf_tab), GFP_KERNEL_ACCOUNT);
 3381		if (!btf_tab)
 3382			return -ENOMEM;
 3383		prog_aux->kfunc_btf_tab = btf_tab;
 3384	}
 3385
 3386	desc_btf = find_kfunc_desc_btf(env, offset);
 3387	if (IS_ERR(desc_btf)) {
 3388		verbose(env, "failed to find BTF for kernel function\n");
 3389		return PTR_ERR(desc_btf);
 3390	}
 3391
 3392	if (find_kfunc_desc(env->prog, func_id, offset))
 3393		return 0;
 3394
 3395	if (tab->nr_descs == MAX_KFUNC_DESCS) {
 3396		verbose(env, "too many different kernel function calls\n");
 3397		return -E2BIG;
 3398	}
 3399
 3400	func = btf_type_by_id(desc_btf, func_id);
 3401	if (!func || !btf_type_is_func(func)) {
 3402		verbose(env, "kernel btf_id %u is not a function\n",
 3403			func_id);
 3404		return -EINVAL;
 3405	}
 3406	func_proto = btf_type_by_id(desc_btf, func->type);
 3407	if (!func_proto || !btf_type_is_func_proto(func_proto)) {
 3408		verbose(env, "kernel function btf_id %u does not have a valid func_proto\n",
 3409			func_id);
 3410		return -EINVAL;
 3411	}
 3412
 3413	func_name = btf_name_by_offset(desc_btf, func->name_off);
 3414	addr = kallsyms_lookup_name(func_name);
 3415	if (!addr) {
 3416		verbose(env, "cannot find address for kernel function %s\n",
 3417			func_name);
 3418		return -EINVAL;
 3419	}
 3420
 3421	if (bpf_dev_bound_kfunc_id(func_id)) {
 3422		err = bpf_dev_bound_kfunc_check(&env->log, prog_aux);
 3423		if (err)
 3424			return err;
 3425	}
 3426
 3427	err = btf_distill_func_proto(&env->log, desc_btf,
 3428				     func_proto, func_name,
 3429				     &func_model);
 3430	if (err)
 3431		return err;
 3432
 3433	desc = &tab->descs[tab->nr_descs++];
 3434	desc->func_id = func_id;
 3435	desc->offset = offset;
 3436	desc->addr = addr;
 3437	desc->func_model = func_model;
 3438	sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
 3439	     kfunc_desc_cmp_by_id_off, NULL);
 3440	return 0;
 3441}
 3442
 3443static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b)
 3444{
 3445	const struct bpf_kfunc_desc *d0 = a;
 3446	const struct bpf_kfunc_desc *d1 = b;
 3447
 3448	if (d0->imm != d1->imm)
 3449		return d0->imm < d1->imm ? -1 : 1;
 3450	if (d0->offset != d1->offset)
 3451		return d0->offset < d1->offset ? -1 : 1;
 3452	return 0;
 3453}
 3454
 3455static int set_kfunc_desc_imm(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc)
 3456{
 3457	unsigned long call_imm;
 3458
 3459	if (bpf_jit_supports_far_kfunc_call()) {
 3460		call_imm = desc->func_id;
 3461	} else {
 3462		call_imm = BPF_CALL_IMM(desc->addr);
 3463		/* Check whether the relative offset overflows desc->imm */
 3464		if ((unsigned long)(s32)call_imm != call_imm) {
 3465			verbose(env, "address of kernel func_id %u is out of range\n",
 3466				desc->func_id);
 3467			return -EINVAL;
 3468		}
 3469	}
 3470	desc->imm = call_imm;
 3471	return 0;
 3472}
 3473
 3474static int sort_kfunc_descs_by_imm_off(struct bpf_verifier_env *env)
 3475{
 3476	struct bpf_kfunc_desc_tab *tab;
 3477	int i, err;
 3478
 3479	tab = env->prog->aux->kfunc_tab;
 3480	if (!tab)
 3481		return 0;
 3482
 3483	for (i = 0; i < tab->nr_descs; i++) {
 3484		err = set_kfunc_desc_imm(env, &tab->descs[i]);
 3485		if (err)
 3486			return err;
 3487	}
 3488
 3489	sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
 3490	     kfunc_desc_cmp_by_imm_off, NULL);
 3491	return 0;
 3492}
 3493
 3494bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
 3495{
 3496	return !!prog->aux->kfunc_tab;
 3497}
 3498
 3499const struct btf_func_model *
 3500bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
 3501			 const struct bpf_insn *insn)
 3502{
 3503	const struct bpf_kfunc_desc desc = {
 3504		.imm = insn->imm,
 3505		.offset = insn->off,
 3506	};
 3507	const struct bpf_kfunc_desc *res;
 3508	struct bpf_kfunc_desc_tab *tab;
 3509
 3510	tab = prog->aux->kfunc_tab;
 3511	res = bsearch(&desc, tab->descs, tab->nr_descs,
 3512		      sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off);
 3513
 3514	return res ? &res->func_model : NULL;
 3515}
 3516
 3517static int add_kfunc_in_insns(struct bpf_verifier_env *env,
 3518			      struct bpf_insn *insn, int cnt)
 3519{
 3520	int i, ret;
 3521
 3522	for (i = 0; i < cnt; i++, insn++) {
 3523		if (bpf_pseudo_kfunc_call(insn)) {
 3524			ret = add_kfunc_call(env, insn->imm, insn->off);
 3525			if (ret < 0)
 3526				return ret;
 3527		}
 3528	}
 3529	return 0;
 3530}
 3531
 3532static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
 3533{
 3534	struct bpf_subprog_info *subprog = env->subprog_info;
 3535	int i, ret, insn_cnt = env->prog->len, ex_cb_insn;
 3536	struct bpf_insn *insn = env->prog->insnsi;
 3537
 3538	/* Add entry function. */
 3539	ret = add_subprog(env, 0);
 3540	if (ret)
 3541		return ret;
 3542
 3543	for (i = 0; i < insn_cnt; i++, insn++) {
 3544		if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) &&
 3545		    !bpf_pseudo_kfunc_call(insn))
 3546			continue;
 3547
 3548		if (!env->bpf_capable) {
 3549			verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
 3550			return -EPERM;
 3551		}
 3552
 3553		if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn))
 3554			ret = add_subprog(env, i + insn->imm + 1);
 3555		else
 3556			ret = add_kfunc_call(env, insn->imm, insn->off);
 3557
 3558		if (ret < 0)
 3559			return ret;
 3560	}
 3561
 3562	ret = bpf_find_exception_callback_insn_off(env);
 3563	if (ret < 0)
 3564		return ret;
 3565	ex_cb_insn = ret;
 3566
 3567	/* If ex_cb_insn > 0, this means that the main program has a subprog
 3568	 * marked using BTF decl tag to serve as the exception callback.
 3569	 */
 3570	if (ex_cb_insn) {
 3571		ret = add_subprog(env, ex_cb_insn);
 3572		if (ret < 0)
 3573			return ret;
 3574		for (i = 1; i < env->subprog_cnt; i++) {
 3575			if (env->subprog_info[i].start != ex_cb_insn)
 3576				continue;
 3577			env->exception_callback_subprog = i;
 3578			mark_subprog_exc_cb(env, i);
 3579			break;
 3580		}
 3581	}
 3582
 3583	/* Add a fake 'exit' subprog which could simplify subprog iteration
 3584	 * logic. 'subprog_cnt' should not be increased.
 3585	 */
 3586	subprog[env->subprog_cnt].start = insn_cnt;
 3587
 3588	if (env->log.level & BPF_LOG_LEVEL2)
 3589		for (i = 0; i < env->subprog_cnt; i++)
 3590			verbose(env, "func#%d @%d\n", i, subprog[i].start);
 3591
 3592	return 0;
 3593}
 3594
 3595static int check_subprogs(struct bpf_verifier_env *env)
 3596{
 3597	int i, subprog_start, subprog_end, off, cur_subprog = 0;
 3598	struct bpf_subprog_info *subprog = env->subprog_info;
 3599	struct bpf_insn *insn = env->prog->insnsi;
 3600	int insn_cnt = env->prog->len;
 3601
 3602	/* now check that all jumps are within the same subprog */
 3603	subprog_start = subprog[cur_subprog].start;
 3604	subprog_end = subprog[cur_subprog + 1].start;
 3605	for (i = 0; i < insn_cnt; i++) {
 3606		u8 code = insn[i].code;
 3607
 3608		if (code == (BPF_JMP | BPF_CALL) &&
 3609		    insn[i].src_reg == 0 &&
 3610		    insn[i].imm == BPF_FUNC_tail_call) {
 3611			subprog[cur_subprog].has_tail_call = true;
 3612			subprog[cur_subprog].tail_call_reachable = true;
 3613		}
 3614		if (BPF_CLASS(code) == BPF_LD &&
 3615		    (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND))
 3616			subprog[cur_subprog].has_ld_abs = true;
 3617		if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32)
 3618			goto next;
 3619		if (BPF_OP(code) == BPF_CALL)
 3620			goto next;
 3621		if (BPF_OP(code) == BPF_EXIT) {
 3622			subprog[cur_subprog].exit_idx = i;
 3623			goto next;
 3624		}
 3625		off = i + bpf_jmp_offset(&insn[i]) + 1;
 3626		if (off < subprog_start || off >= subprog_end) {
 3627			verbose(env, "jump out of range from insn %d to %d\n", i, off);
 3628			return -EINVAL;
 3629		}
 3630next:
 3631		if (i == subprog_end - 1) {
 3632			/* to avoid fall-through from one subprog into another
 3633			 * the last insn of the subprog should be either exit
 3634			 * or unconditional jump back or bpf_throw call
 3635			 */
 3636			if (code != (BPF_JMP | BPF_EXIT) &&
 3637			    code != (BPF_JMP32 | BPF_JA) &&
 3638			    code != (BPF_JMP | BPF_JA)) {
 3639				verbose(env, "last insn is not an exit or jmp\n");
 3640				return -EINVAL;
 3641			}
 3642			subprog_start = subprog_end;
 3643			cur_subprog++;
 3644			if (cur_subprog < env->subprog_cnt)
 3645				subprog_end = subprog[cur_subprog + 1].start;
 3646		}
 3647	}
 3648	return 0;
 3649}
 3650
 3651static int mark_stack_slot_obj_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
 3652				    int spi, int nr_slots)
 3653{
 3654	int err, i;
 3655
 3656	for (i = 0; i < nr_slots; i++) {
 3657		err = bpf_mark_stack_read(env, reg->frameno, env->insn_idx, BIT(spi - i));
 3658		if (err)
 3659			return err;
 3660		mark_stack_slot_scratched(env, spi - i);
 3661	}
 3662	return 0;
 3663}
 3664
 3665static int mark_dynptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
 3666{
 3667	int spi;
 3668
 3669	/* For CONST_PTR_TO_DYNPTR, it must have already been done by
 3670	 * check_reg_arg in check_helper_call and mark_btf_func_reg_size in
 3671	 * check_kfunc_call.
 3672	 */
 3673	if (reg->type == CONST_PTR_TO_DYNPTR)
 3674		return 0;
 3675	spi = dynptr_get_spi(env, reg);
 3676	if (spi < 0)
 3677		return spi;
 3678	/* Caller ensures dynptr is valid and initialized, which means spi is in
 3679	 * bounds and spi is the first dynptr slot. Simply mark stack slot as
 3680	 * read.
 3681	 */
 3682	return mark_stack_slot_obj_read(env, reg, spi, BPF_DYNPTR_NR_SLOTS);
 3683}
 3684
 3685static int mark_iter_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
 3686			  int spi, int nr_slots)
 3687{
 3688	return mark_stack_slot_obj_read(env, reg, spi, nr_slots);
 3689}
 3690
 3691static int mark_irq_flag_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
 3692{
 3693	int spi;
 3694
 3695	spi = irq_flag_get_spi(env, reg);
 3696	if (spi < 0)
 3697		return spi;
 3698	return mark_stack_slot_obj_read(env, reg, spi, 1);
 3699}
 3700
 3701/* This function is supposed to be used by the following 32-bit optimization
 3702 * code only. It returns TRUE if the source or destination register operates
 3703 * on 64-bit, otherwise return FALSE.
 3704 */
 3705static bool is_reg64(struct bpf_insn *insn,
 3706		     u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t)
 3707{
 3708	u8 code, class, op;
 3709
 3710	code = insn->code;
 3711	class = BPF_CLASS(code);
 3712	op = BPF_OP(code);
 3713	if (class == BPF_JMP) {
 3714		/* BPF_EXIT for "main" will reach here. Return TRUE
 3715		 * conservatively.
 3716		 */
 3717		if (op == BPF_EXIT)
 3718			return true;
 3719		if (op == BPF_CALL) {
 3720			/* BPF to BPF call will reach here because of marking
 3721			 * caller saved clobber with DST_OP_NO_MARK for which we
 3722			 * don't care the register def because they are anyway
 3723			 * marked as NOT_INIT already.
 3724			 */
 3725			if (insn->src_reg == BPF_PSEUDO_CALL)
 3726				return false;
 3727			/* Helper call will reach here because of arg type
 3728			 * check, conservatively return TRUE.
 3729			 */
 3730			if (t == SRC_OP)
 3731				return true;
 3732
 3733			return false;
 3734		}
 3735	}
 3736
 3737	if (class == BPF_ALU64 && op == BPF_END && (insn->imm == 16 || insn->imm == 32))
 3738		return false;
 3739
 3740	if (class == BPF_ALU64 || class == BPF_JMP ||
 3741	    (class == BPF_ALU && op == BPF_END && insn->imm == 64))
 3742		return true;
 3743
 3744	if (class == BPF_ALU || class == BPF_JMP32)
 3745		return false;
 3746
 3747	if (class == BPF_LDX) {
 3748		if (t != SRC_OP)
 3749			return BPF_SIZE(code) == BPF_DW || BPF_MODE(code) == BPF_MEMSX;
 3750		/* LDX source must be ptr. */
 3751		return true;
 3752	}
 3753
 3754	if (class == BPF_STX) {
 3755		/* BPF_STX (including atomic variants) has one or more source
 3756		 * operands, one of which is a ptr. Check whether the caller is
 3757		 * asking about it.
 3758		 */
 3759		if (t == SRC_OP && reg->type != SCALAR_VALUE)
 3760			return true;
 3761		return BPF_SIZE(code) == BPF_DW;
 3762	}
 3763
 3764	if (class == BPF_LD) {
 3765		u8 mode = BPF_MODE(code);
 3766
 3767		/* LD_IMM64 */
 3768		if (mode == BPF_IMM)
 3769			return true;
 3770
 3771		/* Both LD_IND and LD_ABS return 32-bit data. */
 3772		if (t != SRC_OP)
 3773			return  false;
 3774
 3775		/* Implicit ctx ptr. */
 3776		if (regno == BPF_REG_6)
 3777			return true;
 3778
 3779		/* Explicit source could be any width. */
 3780		return true;
 3781	}
 3782
 3783	if (class == BPF_ST)
 3784		/* The only source register for BPF_ST is a ptr. */
 3785		return true;
 3786
 3787	/* Conservatively return true at default. */
 3788	return true;
 3789}
 3790
 3791/* Return the regno defined by the insn, or -1. */
 3792static int insn_def_regno(const struct bpf_insn *insn)
 3793{
 3794	switch (BPF_CLASS(insn->code)) {
 3795	case BPF_JMP:
 3796	case BPF_JMP32:
 3797	case BPF_ST:
 3798		return -1;
 3799	case BPF_STX:
 3800		if (BPF_MODE(insn->code) == BPF_ATOMIC ||
 3801		    BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) {
 3802			if (insn->imm == BPF_CMPXCHG)
 3803				return BPF_REG_0;
 3804			else if (insn->imm == BPF_LOAD_ACQ)
 3805				return insn->dst_reg;
 3806			else if (insn->imm & BPF_FETCH)
 3807				return insn->src_reg;
 3808		}
 3809		return -1;
 3810	default:
 3811		return insn->dst_reg;
 3812	}
 3813}
 3814
 3815/* Return TRUE if INSN has defined any 32-bit value explicitly. */
 3816static bool insn_has_def32(struct bpf_insn *insn)
 3817{
 3818	int dst_reg = insn_def_regno(insn);
 3819
 3820	if (dst_reg == -1)
 3821		return false;
 3822
 3823	return !is_reg64(insn, dst_reg, NULL, DST_OP);
 3824}
 3825
 3826static void mark_insn_zext(struct bpf_verifier_env *env,
 3827			   struct bpf_reg_state *reg)
 3828{
 3829	s32 def_idx = reg->subreg_def;
 3830
 3831	if (def_idx == DEF_NOT_SUBREG)
 3832		return;
 3833
 3834	env->insn_aux_data[def_idx - 1].zext_dst = true;
 3835	/* The dst will be zero extended, so won't be sub-register anymore. */
 3836	reg->subreg_def = DEF_NOT_SUBREG;
 3837}
 3838
 3839static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno,
 3840			   enum reg_arg_type t)
 3841{
 3842	struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;
 3843	struct bpf_reg_state *reg;
 3844	bool rw64;
 3845
 3846	if (regno >= MAX_BPF_REG) {
 3847		verbose(env, "R%d is invalid\n", regno);
 3848		return -EINVAL;
 3849	}
 3850
 3851	mark_reg_scratched(env, regno);
 3852
 3853	reg = &regs[regno];
 3854	rw64 = is_reg64(insn, regno, reg, t);
 3855	if (t == SRC_OP) {
 3856		/* check whether register used as source operand can be read */
 3857		if (reg->type == NOT_INIT) {
 3858			verbose(env, "R%d !read_ok\n", regno);
 3859			return -EACCES;
 3860		}
 3861		/* We don't need to worry about FP liveness because it's read-only */
 3862		if (regno == BPF_REG_FP)
 3863			return 0;
 3864
 3865		if (rw64)
 3866			mark_insn_zext(env, reg);
 3867
 3868		return 0;
 3869	} else {
 3870		/* check whether register used as dest operand can be written to */
 3871		if (regno == BPF_REG_FP) {
 3872			verbose(env, "frame pointer is read only\n");
 3873			return -EACCES;
 3874		}
 3875		reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1;
 3876		if (t == DST_OP)
 3877			mark_reg_unknown(env, regs, regno);
 3878	}
 3879	return 0;
 3880}
 3881
 3882static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,
 3883			 enum reg_arg_type t)
 3884{
 3885	struct bpf_verifier_state *vstate = env->cur_state;
 3886	struct bpf_func_state *state = vstate->frame[vstate->curframe];
 3887
 3888	return __check_reg_arg(env, state->regs, regno, t);
 3889}
 3890
 3891static int insn_stack_access_flags(int frameno, int spi)
 3892{
 3893	return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno;
 3894}
 3895
 3896static int insn_stack_access_spi(int insn_flags)
 3897{
 3898	return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK;
 3899}
 3900
 3901static int insn_stack_access_frameno(int insn_flags)
 3902{
 3903	return insn_flags & INSN_F_FRAMENO_MASK;
 3904}
 3905
 3906static void mark_jmp_point(struct bpf_verifier_env *env, int idx)
 3907{
 3908	env->insn_aux_data[idx].jmp_point = true;
 3909}
 3910
 3911static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx)
 3912{
 3913	return env->insn_aux_data[insn_idx].jmp_point;
 3914}
 3915
 3916#define LR_FRAMENO_BITS	3
 3917#define LR_SPI_BITS	6
 3918#define LR_ENTRY_BITS	(LR_SPI_BITS + LR_FRAMENO_BITS + 1)
 3919#define LR_SIZE_BITS	4
 3920#define LR_FRAMENO_MASK	((1ull << LR_FRAMENO_BITS) - 1)
 3921#define LR_SPI_MASK	((1ull << LR_SPI_BITS)     - 1)
 3922#define LR_SIZE_MASK	((1ull << LR_SIZE_BITS)    - 1)
 3923#define LR_SPI_OFF	LR_FRAMENO_BITS
 3924#define LR_IS_REG_OFF	(LR_SPI_BITS + LR_FRAMENO_BITS)
 3925#define LINKED_REGS_MAX	6
 3926
 3927struct linked_reg {
 3928	u8 frameno;
 3929	union {
 3930		u8 spi;
 3931		u8 regno;
 3932	};
 3933	bool is_reg;
 3934};
 3935
 3936struct linked_regs {
 3937	int cnt;
 3938	struct linked_reg entries[LINKED_REGS_MAX];
 3939};
 3940
 3941static struct linked_reg *linked_regs_push(struct linked_regs *s)
 3942{
 3943	if (s->cnt < LINKED_REGS_MAX)
 3944		return &s->entries[s->cnt++];
 3945
 3946	return NULL;
 3947}
 3948
 3949/* Use u64 as a vector of 6 10-bit values, use first 4-bits to track
 3950 * number of elements currently in stack.
 3951 * Pack one history entry for linked registers as 10 bits in the following format:
 3952 * - 3-bits frameno
 3953 * - 6-bits spi_or_reg
 3954 * - 1-bit  is_reg
 3955 */
 3956static u64 linked_regs_pack(struct linked_regs *s)
 3957{
 3958	u64 val = 0;
 3959	int i;
 3960
 3961	for (i = 0; i < s->cnt; ++i) {
 3962		struct linked_reg *e = &s->entries[i];
 3963		u64 tmp = 0;
 3964
 3965		tmp |= e->frameno;
 3966		tmp |= e->spi << LR_SPI_OFF;
 3967		tmp |= (e->is_reg ? 1 : 0) << LR_IS_REG_OFF;
 3968
 3969		val <<= LR_ENTRY_BITS;
 3970		val |= tmp;
 3971	}
 3972	val <<= LR_SIZE_BITS;
 3973	val |= s->cnt;
 3974	return val;
 3975}
 3976
 3977static void linked_regs_unpack(u64 val, struct linked_regs *s)
 3978{
 3979	int i;
 3980
 3981	s->cnt = val & LR_SIZE_MASK;
 3982	val >>= LR_SIZE_BITS;
 3983
 3984	for (i = 0; i < s->cnt; ++i) {
 3985		struct linked_reg *e = &s->entries[i];
 3986
 3987		e->frameno =  val & LR_FRAMENO_MASK;
 3988		e->spi     = (val >> LR_SPI_OFF) & LR_SPI_MASK;
 3989		e->is_reg  = (val >> LR_IS_REG_OFF) & 0x1;
 3990		val >>= LR_ENTRY_BITS;
 3991	}
 3992}
 3993
 3994/* for any branch, call, exit record the history of jmps in the given state */
 3995static int push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
 3996			    int insn_flags, u64 linked_regs)
 3997{
 3998	u32 cnt = cur->jmp_history_cnt;
 3999	struct bpf_jmp_history_entry *p;
 4000	size_t alloc_size;
 4001
 4002	/* combine instruction flags if we already recorded this instruction */
 4003	if (env->cur_hist_ent) {
 4004		/* atomic instructions push insn_flags twice, for READ and
 4005		 * WRITE sides, but they should agree on stack slot
 4006		 */
 4007		verifier_bug_if((env->cur_hist_ent->flags & insn_flags) &&
 4008				(env->cur_hist_ent->flags & insn_flags) != insn_flags,
 4009				env, "insn history: insn_idx %d cur flags %x new flags %x",
 4010				env->insn_idx, env->cur_hist_ent->flags, insn_flags);
 4011		env->cur_hist_ent->flags |= insn_flags;
 4012		verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env,
 4013				"insn history: insn_idx %d linked_regs: %#llx",
 4014				env->insn_idx, env->cur_hist_ent->linked_regs);
 4015		env->cur_hist_ent->linked_regs = linked_regs;
 4016		return 0;
 4017	}
 4018
 4019	cnt++;
 4020	alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p)));
 4021	p = krealloc(cur->jmp_history, alloc_size, GFP_KERNEL_ACCOUNT);
 4022	if (!p)
 4023		return -ENOMEM;
 4024	cur->jmp_history = p;
 4025
 4026	p = &cur->jmp_history[cnt - 1];
 4027	p->idx = env->insn_idx;
 4028	p->prev_idx = env->prev_insn_idx;
 4029	p->flags = insn_flags;
 4030	p->linked_regs = linked_regs;
 4031	cur->jmp_history_cnt = cnt;
 4032	env->cur_hist_ent = p;
 4033
 4034	return 0;
 4035}
 4036
 4037static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st,
 4038						        u32 hist_end, int insn_idx)
 4039{
 4040	if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx)
 4041		return &st->jmp_history[hist_end - 1];
 4042	return NULL;
 4043}
 4044
 4045/* Backtrack one insn at a time. If idx is not at the top of recorded
 4046 * history then previous instruction came from straight line execution.
 4047 * Return -ENOENT if we exhausted all instructions within given state.
 4048 *
 4049 * It's legal to have a bit of a looping with the same starting and ending
 4050 * insn index within the same state, e.g.: 3->4->5->3, so just because current
 4051 * instruction index is the same as state's first_idx doesn't mean we are
 4052 * done. If there is still some jump history left, we should keep going. We
 4053 * need to take into account that we might have a jump history between given
 4054 * state's parent and itself, due to checkpointing. In this case, we'll have
 4055 * history entry recording a jump from last instruction of parent state and
 4056 * first instruction of given state.
 4057 */
 4058static int get_prev_insn_idx(struct bpf_verifier_state *st, int i,
 4059			     u32 *history)
 4060{
 4061	u32 cnt = *history;
 4062
 4063	if (i == st->first_insn_idx) {
 4064		if (cnt == 0)
 4065			return -ENOENT;
 4066		if (cnt == 1 && st->jmp_history[0].idx == i)
 4067			return -ENOENT;
 4068	}
 4069
 4070	if (cnt && st->jmp_history[cnt - 1].idx == i) {
 4071		i = st->jmp_history[cnt - 1].prev_idx;
 4072		(*history)--;
 4073	} else {
 4074		i--;
 4075	}
 4076	return i;
 4077}
 4078
 4079static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn)
 4080{
 4081	const struct btf_type *func;
 4082	struct btf *desc_btf;
 4083
 4084	if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL)
 4085		return NULL;
 4086
 4087	desc_btf = find_kfunc_desc_btf(data, insn->off);
 4088	if (IS_ERR(desc_btf))
 4089		return "<error>";
 4090
 4091	func = btf_type_by_id(desc_btf, insn->imm);
 4092	return btf_name_by_offset(desc_btf, func->name_off);
 4093}
 4094
 4095static void verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn)
 4096{
 4097	const struct bpf_insn_cbs cbs = {
 4098		.cb_call	= disasm_kfunc_name,
 4099		.cb_print	= verbose,
 4100		.private_data	= env,
 4101	};
 4102
 4103	print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
 4104}
 4105
 4106static inline void bt_init(struct backtrack_state *bt, u32 frame)
 4107{
 4108	bt->frame = frame;
 4109}
 4110
 4111static inline void bt_reset(struct backtrack_state *bt)
 4112{
 4113	struct bpf_verifier_env *env = bt->env;
 4114
 4115	memset(bt, 0, sizeof(*bt));
 4116	bt->env = env;
 4117}
 4118
 4119static inline u32 bt_empty(struct backtrack_state *bt)
 4120{
 4121	u64 mask = 0;
 4122	int i;
 4123
 4124	for (i = 0; i <= bt->frame; i++)
 4125		mask |= bt->reg_masks[i] | bt->stack_masks[i];
 4126
 4127	return mask == 0;
 4128}
 4129
 4130static inline int bt_subprog_enter(struct backtrack_state *bt)
 4131{
 4132	if (bt->frame == MAX_CALL_FRAMES - 1) {
 4133		verifier_bug(bt->env, "subprog enter from frame %d", bt->frame);
 4134		return -EFAULT;
 4135	}
 4136	bt->frame++;
 4137	return 0;
 4138}
 4139
 4140static inline int bt_subprog_exit(struct backtrack_state *bt)
 4141{
 4142	if (bt->frame == 0) {
 4143		verifier_bug(bt->env, "subprog exit from frame 0");
 4144		return -EFAULT;
 4145	}
 4146	bt->frame--;
 4147	return 0;
 4148}
 4149
 4150static inline void bt_set_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg)
 4151{
 4152	bt->reg_masks[frame] |= 1 << reg;
 4153}
 4154
 4155static inline void bt_clear_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg)
 4156{
 4157	bt->reg_masks[frame] &= ~(1 << reg);
 4158}
 4159
 4160static inline void bt_set_reg(struct backtrack_state *bt, u32 reg)
 4161{
 4162	bt_set_frame_reg(bt, bt->frame, reg);
 4163}
 4164
 4165static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg)
 4166{
 4167	bt_clear_frame_reg(bt, bt->frame, reg);
 4168}
 4169
 4170static inline void bt_set_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)
 4171{
 4172	bt->stack_masks[frame] |= 1ull << slot;
 4173}
 4174
 4175static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)
 4176{
 4177	bt->stack_masks[frame] &= ~(1ull << slot);
 4178}
 4179
 4180static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame)
 4181{
 4182	return bt->reg_masks[frame];
 4183}
 4184
 4185static inline u32 bt_reg_mask(struct backtrack_state *bt)
 4186{
 4187	return bt->reg_masks[bt->frame];
 4188}
 4189
 4190static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame)
 4191{
 4192	return bt->stack_masks[frame];
 4193}
 4194
 4195static inline u64 bt_stack_mask(struct backtrack_state *bt)
 4196{
 4197	return bt->stack_masks[bt->frame];
 4198}
 4199
 4200static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg)
 4201{
 4202	return bt->reg_masks[bt->frame] & (1 << reg);
 4203}
 4204
 4205static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u32 reg)
 4206{
 4207	return bt->reg_masks[frame] & (1 << reg);
 4208}
 4209
 4210static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot)
 4211{
 4212	return bt->stack_masks[frame] & (1ull << slot);
 4213}
 4214
 4215/* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */
 4216static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask)
 4217{
 4218	DECLARE_BITMAP(mask, 64);
 4219	bool first = true;
 4220	int i, n;
 4221
 4222	buf[0] = '\0';
 4223
 4224	bitmap_from_u64(mask, reg_mask);
 4225	for_each_set_bit(i, mask, 32) {
 4226		n = snprintf(buf, buf_sz, "%sr%d", first ? "" : ",", i);
 4227		first = false;
 4228		buf += n;
 4229		buf_sz -= n;
 4230		if (buf_sz < 0)
 4231			break;
 4232	}
 4233}
 4234/* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */
 4235void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask)
 4236{
 4237	DECLARE_BITMAP(mask, 64);
 4238	bool first = true;
 4239	int i, n;
 4240
 4241	buf[0] = '\0';
 4242
 4243	bitmap_from_u64(mask, stack_mask);
 4244	for_each_set_bit(i, mask, 64) {
 4245		n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8);
 4246		first = false;
 4247		buf += n;
 4248		buf_sz -= n;
 4249		if (buf_sz < 0)
 4250			break;
 4251	}
 4252}
 4253
 4254/* If any register R in hist->linked_regs is marked as precise in bt,
 4255 * do bt_set_frame_{reg,slot}(bt, R) for all registers in hist->linked_regs.
 4256 */
 4257static void bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist)
 4258{
 4259	struct linked_regs linked_regs;
 4260	bool some_precise = false;
 4261	int i;
 4262
 4263	if (!hist || hist->linked_regs == 0)
 4264		return;
 4265
 4266	linked_regs_unpack(hist->linked_regs, &linked_regs);
 4267	for (i = 0; i < linked_regs.cnt; ++i) {
 4268		struct linked_reg *e = &linked_regs.entries[i];
 4269
 4270		if ((e->is_reg && bt_is_frame_reg_set(bt, e->frameno, e->regno)) ||
 4271		    (!e->is_reg && bt_is_frame_slot_set(bt, e->frameno, e->spi))) {
 4272			some_precise = true;
 4273			break;
 4274		}
 4275	}
 4276
 4277	if (!some_precise)
 4278		return;
 4279
 4280	for (i = 0; i < linked_regs.cnt; ++i) {
 4281		struct linked_reg *e = &linked_regs.entries[i];
 4282
 4283		if (e->is_reg)
 4284			bt_set_frame_reg(bt, e->frameno, e->regno);
 4285		else
 4286			bt_set_frame_slot(bt, e->frameno, e->spi);
 4287	}
 4288}
 4289
 4290/* For given verifier state backtrack_insn() is called from the last insn to
 4291 * the first insn. Its purpose is to compute a bitmask of registers and
 4292 * stack slots that needs precision in the parent verifier state.
 4293 *
 4294 * @idx is an index of the instruction we are currently processing;
 4295 * @subseq_idx is an index of the subsequent instruction that:
 4296 *   - *would be* executed next, if jump history is viewed in forward order;
 4297 *   - *was* processed previously during backtracking.
 4298 */
 4299static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
 4300			  struct bpf_jmp_history_entry *hist, struct backtrack_state *bt)
 4301{
 4302	struct bpf_insn *insn = env->prog->insnsi + idx;
 4303	u8 class = BPF_CLASS(insn->code);
 4304	u8 opcode = BPF_OP(insn->code);
 4305	u8 mode = BPF_MODE(insn->code);
 4306	u32 dreg = insn->dst_reg;
 4307	u32 sreg = insn->src_reg;
 4308	u32 spi, i, fr;
 4309
 4310	if (insn->code == 0)
 4311		return 0;
 4312	if (env->log.level & BPF_LOG_LEVEL2) {
 4313		fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt));
 4314		verbose(env, "mark_precise: frame%d: regs=%s ",
 4315			bt->frame, env->tmp_str_buf);
 4316		bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));
 4317		verbose(env, "stack=%s before ", env->tmp_str_buf);
 4318		verbose(env, "%d: ", idx);
 4319		verbose_insn(env, insn);
 4320	}
 4321
 4322	/* If there is a history record that some registers gained range at this insn,
 4323	 * propagate precision marks to those registers, so that bt_is_reg_set()
 4324	 * accounts for these registers.
 4325	 */
 4326	bt_sync_linked_regs(bt, hist);
 4327
 4328	if (class == BPF_ALU || class == BPF_ALU64) {
 4329		if (!bt_is_reg_set(bt, dreg))
 4330			return 0;
 4331		if (opcode == BPF_END || opcode == BPF_NEG) {
 4332			/* sreg is reserved and unused
 4333			 * dreg still need precision before this insn
 4334			 */
 4335			return 0;
 4336		} else if (opcode == BPF_MOV) {
 4337			if (BPF_SRC(insn->code) == BPF_X) {
 4338				/* dreg = sreg or dreg = (s8, s16, s32)sreg
 4339				 * dreg needs precision after this insn
 4340				 * sreg needs precision before this insn
 4341				 */
 4342				bt_clear_reg(bt, dreg);
 4343				if (sreg != BPF_REG_FP)
 4344					bt_set_reg(bt, sreg);
 4345			} else {
 4346				/* dreg = K
 4347				 * dreg needs precision after this insn.
 4348				 * Corresponding register is already marked
 4349				 * as precise=true in this verifier state.
 4350				 * No further markings in parent are necessary
 4351				 */
 4352				bt_clear_reg(bt, dreg);
 4353			}
 4354		} else {
 4355			if (BPF_SRC(insn->code) == BPF_X) {
 4356				/* dreg += sreg
 4357				 * both dreg and sreg need precision
 4358				 * before this insn
 4359				 */
 4360				if (sreg != BPF_REG_FP)
 4361					bt_set_reg(bt, sreg);
 4362			} /* else dreg += K
 4363			   * dreg still needs precision before this insn
 4364			   */
 4365		}
 4366	} else if (class == BPF_LDX ||
 4367		   is_atomic_load_insn(insn) ||
 4368		   is_atomic_fetch_insn(insn)) {
 4369		u32 load_reg = dreg;
 4370
 4371		/*
 4372		 * Atomic fetch operation writes the old value into
 4373		 * a register (sreg or r0) and if it was tracked for
 4374		 * precision, propagate to the stack slot like we do
 4375		 * in regular ldx.
 4376		 */
 4377		if (is_atomic_fetch_insn(insn))
 4378			load_reg = insn->imm == BPF_CMPXCHG ?
 4379				   BPF_REG_0 : sreg;
 4380
 4381		if (!bt_is_reg_set(bt, load_reg))
 4382			return 0;
 4383		bt_clear_reg(bt, load_reg);
 4384
 4385		/* scalars can only be spilled into stack w/o losing precision.
 4386		 * Load from any other memory can be zero extended.
 4387		 * The desire to keep that precision is already indicated
 4388		 * by 'precise' mark in corresponding register of this state.
 4389		 * No further tracking necessary.
 4390		 */
 4391		if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
 4392			return 0;
 4393		/* dreg = *(u64 *)[fp - off] was a fill from the stack.
 4394		 * that [fp - off] slot contains scalar that needs to be
 4395		 * tracked with precision
 4396		 */
 4397		spi = insn_stack_access_spi(hist->flags);
 4398		fr = insn_stack_access_frameno(hist->flags);
 4399		bt_set_frame_slot(bt, fr, spi);
 4400	} else if (class == BPF_STX || class == BPF_ST) {
 4401		if (bt_is_reg_set(bt, dreg))
 4402			/* stx & st shouldn't be using _scalar_ dst_reg
 4403			 * to access memory. It means backtracking
 4404			 * encountered a case of pointer subtraction.
 4405			 */
 4406			return -ENOTSUPP;
 4407		/* scalars can only be spilled into stack */
 4408		if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
 4409			return 0;
 4410		spi = insn_stack_access_spi(hist->flags);
 4411		fr = insn_stack_access_frameno(hist->flags);
 4412		if (!bt_is_frame_slot_set(bt, fr, spi))
 4413			return 0;
 4414		bt_clear_frame_slot(bt, fr, spi);
 4415		if (class == BPF_STX)
 4416			bt_set_reg(bt, sreg);
 4417	} else if (class == BPF_JMP || class == BPF_JMP32) {
 4418		if (bpf_pseudo_call(insn)) {
 4419			int subprog_insn_idx, subprog;
 4420
 4421			subprog_insn_idx = idx + insn->imm + 1;
 4422			subprog = find_subprog(env, subprog_insn_idx);
 4423			if (subprog < 0)
 4424				return -EFAULT;
 4425
 4426			if (subprog_is_global(env, subprog)) {
 4427				/* check that jump history doesn't have any
 4428				 * extra instructions from subprog; the next
 4429				 * instruction after call to global subprog
 4430				 * should be literally next instruction in
 4431				 * caller program
 4432				 */
 4433				verifier_bug_if(idx + 1 != subseq_idx, env,
 4434						"extra insn from subprog");
 4435				/* r1-r5 are invalidated after subprog call,
 4436				 * so for global func call it shouldn't be set
 4437				 * anymore
 4438				 */
 4439				if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
 4440					verifier_bug(env, "global subprog unexpected regs %x",
 4441						     bt_reg_mask(bt));
 4442					return -EFAULT;
 4443				}
 4444				/* global subprog always sets R0 */
 4445				bt_clear_reg(bt, BPF_REG_0);
 4446				return 0;
 4447			} else {
 4448				/* static subprog call instruction, which
 4449				 * means that we are exiting current subprog,
 4450				 * so only r1-r5 could be still requested as
 4451				 * precise, r0 and r6-r10 or any stack slot in
 4452				 * the current frame should be zero by now
 4453				 */
 4454				if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
 4455					verifier_bug(env, "static subprog unexpected regs %x",
 4456						     bt_reg_mask(bt));
 4457					return -EFAULT;
 4458				}
 4459				/* we are now tracking register spills correctly,
 4460				 * so any instance of leftover slots is a bug
 4461				 */
 4462				if (bt_stack_mask(bt) != 0) {
 4463					verifier_bug(env,
 4464						     "static subprog leftover stack slots %llx",
 4465						     bt_stack_mask(bt));
 4466					return -EFAULT;
 4467				}
 4468				/* propagate r1-r5 to the caller */
 4469				for (i = BPF_REG_1; i <= BPF_REG_5; i++) {
 4470					if (bt_is_reg_set(bt, i)) {
 4471						bt_clear_reg(bt, i);
 4472						bt_set_frame_reg(bt, bt->frame - 1, i);
 4473					}
 4474				}
 4475				if (bt_subprog_exit(bt))
 4476					return -EFAULT;
 4477				return 0;
 4478			}
 4479		} else if (is_sync_callback_calling_insn(insn) && idx != subseq_idx - 1) {
 4480			/* exit from callback subprog to callback-calling helper or
 4481			 * kfunc call. Use idx/subseq_idx check to discern it from
 4482			 * straight line code backtracking.
 4483			 * Unlike the subprog call handling above, we shouldn't
 4484			 * propagate precision of r1-r5 (if any requested), as they are
 4485			 * not actually arguments passed directly to callback subprogs
 4486			 */
 4487			if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
 4488				verifier_bug(env, "callback unexpected regs %x",
 4489					     bt_reg_mask(bt));
 4490				return -EFAULT;
 4491			}
 4492			if (bt_stack_mask(bt) != 0) {
 4493				verifier_bug(env, "callback leftover stack slots %llx",
 4494					     bt_stack_mask(bt));
 4495				return -EFAULT;
 4496			}
 4497			/* clear r1-r5 in callback subprog's mask */
 4498			for (i = BPF_REG_1; i <= BPF_REG_5; i++)
 4499				bt_clear_reg(bt, i);
 4500			if (bt_subprog_exit(bt))
 4501				return -EFAULT;
 4502			return 0;
 4503		} else if (opcode == BPF_CALL) {
 4504			/* kfunc with imm==0 is invalid and fixup_kfunc_call will
 4505			 * catch this error later. Make backtracking conservative
 4506			 * with ENOTSUPP.
 4507			 */
 4508			if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && insn->imm == 0)
 4509				return -ENOTSUPP;
 4510			/* regular helper call sets R0 */
 4511			bt_clear_reg(bt, BPF_REG_0);
 4512			if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
 4513				/* if backtracking was looking for registers R1-R5
 4514				 * they should have been found already.
 4515				 */
 4516				verifier_bug(env, "backtracking call unexpected regs %x",
 4517					     bt_reg_mask(bt));
 4518				return -EFAULT;
 4519			}
 4520			if (insn->src_reg == BPF_REG_0 && insn->imm == BPF_FUNC_tail_call
 4521			    && subseq_idx - idx != 1) {
 4522				if (bt_subprog_enter(bt))
 4523					return -EFAULT;
 4524			}
 4525		} else if (opcode == BPF_EXIT) {
 4526			bool r0_precise;
 4527
 4528			/* Backtracking to a nested function call, 'idx' is a part of
 4529			 * the inner frame 'subseq_idx' is a part of the outer frame.
 4530			 * In case of a regular function call, instructions giving
 4531			 * precision to registers R1-R5 should have been found already.
 4532			 * In case of a callback, it is ok to have R1-R5 marked for
 4533			 * backtracking, as these registers are set by the function
 4534			 * invoking callback.
 4535			 */
 4536			if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx))
 4537				for (i = BPF_REG_1; i <= BPF_REG_5; i++)
 4538					bt_clear_reg(bt, i);
 4539			if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
 4540				verifier_bug(env, "backtracking exit unexpected regs %x",
 4541					     bt_reg_mask(bt));
 4542				return -EFAULT;
 4543			}
 4544
 4545			/* BPF_EXIT in subprog or callback always returns
 4546			 * right after the call instruction, so by checking
 4547			 * whether the instruction at subseq_idx-1 is subprog
 4548			 * call or not we can distinguish actual exit from
 4549			 * *subprog* from exit from *callback*. In the former
 4550			 * case, we need to propagate r0 precision, if
 4551			 * necessary. In the former we never do that.
 4552			 */
 4553			r0_precise = subseq_idx - 1 >= 0 &&
 4554				     bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) &&
 4555				     bt_is_reg_set(bt, BPF_REG_0);
 4556
 4557			bt_clear_reg(bt, BPF_REG_0);
 4558			if (bt_subprog_enter(bt))
 4559				return -EFAULT;
 4560
 4561			if (r0_precise)
 4562				bt_set_reg(bt, BPF_REG_0);
 4563			/* r6-r9 and stack slots will stay set in caller frame
 4564			 * bitmasks until we return back from callee(s)
 4565			 */
 4566			return 0;
 4567		} else if (BPF_SRC(insn->code) == BPF_X) {
 4568			if (!bt_is_reg_set(bt, dreg) && !bt_is_reg_set(bt, sreg))
 4569				return 0;
 4570			/* dreg <cond> sreg
 4571			 * Both dreg and sreg need precision before
 4572			 * this insn. If only sreg was marked precise
 4573			 * before it would be equally necessary to
 4574			 * propagate it to dreg.
 4575			 */
 4576			if (!hist || !(hist->flags & INSN_F_SRC_REG_STACK))
 4577				bt_set_reg(bt, sreg);
 4578			if (!hist || !(hist->flags & INSN_F_DST_REG_STACK))
 4579				bt_set_reg(bt, dreg);
 4580		} else if (BPF_SRC(insn->code) == BPF_K) {
 4581			 /* dreg <cond> K
 4582			  * Only dreg still needs precision before
 4583			  * this insn, so for the K-based conditional
 4584			  * there is nothing new to be marked.
 4585			  */
 4586		}
 4587	} else if (class == BPF_LD) {
 4588		if (!bt_is_reg_set(bt, dreg))
 4589			return 0;
 4590		bt_clear_reg(bt, dreg);
 4591		/* It's ld_imm64 or ld_abs or ld_ind.
 4592		 * For ld_imm64 no further tracking of precision
 4593		 * into parent is necessary
 4594		 */
 4595		if (mode == BPF_IND || mode == BPF_ABS)
 4596			/* to be analyzed */
 4597			return -ENOTSUPP;
 4598	}
 4599	/* Propagate precision marks to linked registers, to account for
 4600	 * registers marked as precise in this function.
 4601	 */
 4602	bt_sync_linked_regs(bt, hist);
 4603	return 0;
 4604}
 4605
 4606/* the scalar precision tracking algorithm:
 4607 * . at the start all registers have precise=false.
 4608 * . scalar ranges are tracked as normal through alu and jmp insns.
 4609 * . once precise value of the scalar register is used in:
 4610 *   .  ptr + scalar alu
 4611 *   . if (scalar cond K|scalar)
 4612 *   .  helper_call(.., scalar, ...) where ARG_CONST is expected
 4613 *   backtrack through the verifier states and mark all registers and
 4614 *   stack slots with spilled constants that these scalar registers
 4615 *   should be precise.
 4616 * . during state pruning two registers (or spilled stack slots)
 4617 *   are equivalent if both are not precise.
 4618 *
 4619 * Note the verifier cannot simply walk register parentage chain,
 4620 * since many different registers and stack slots could have been
 4621 * used to compute single precise scalar.
 4622 *
 4623 * The approach of starting with precise=true for all registers and then
 4624 * backtrack to mark a register as not precise when the verifier detects
 4625 * that program doesn't care about specific value (e.g., when helper
 4626 * takes register as ARG_ANYTHING parameter) is not safe.
 4627 *
 4628 * It's ok to walk single parentage chain of the verifier states.
 4629 * It's possible that this backtracking will go all the way till 1st insn.
 4630 * All other branches will be explored for needing precision later.
 4631 *
 4632 * The backtracking needs to deal with cases like:
 4633 *   R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0)
 4634 * r9 -= r8
 4635 * r5 = r9
 4636 * if r5 > 0x79f goto pc+7
 4637 *    R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff))
 4638 * r5 += 1
 4639 * ...
 4640 * call bpf_perf_event_output#25
 4641 *   where .arg5_type = ARG_CONST_SIZE_OR_ZERO
 4642 *
 4643 * and this case:
 4644 * r6 = 1
 4645 * call foo // uses callee's r6 inside to compute r0
 4646 * r0 += r6
 4647 * if r0 == 0 goto
 4648 *
 4649 * to track above reg_mask/stack_mask needs to be independent for each frame.
 4650 *
 4651 * Also if parent's curframe > frame where backtracking started,
 4652 * the verifier need to mark registers in both frames, otherwise callees
 4653 * may incorrectly prune callers. This is similar to
 4654 * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences")
 4655 *
 4656 * For now backtracking falls back into conservative marking.
 4657 */
 4658static void mark_all_scalars_precise(struct bpf_verifier_env *env,
 4659				     struct bpf_verifier_state *st)
 4660{
 4661	struct bpf_func_state *func;
 4662	struct bpf_reg_state *reg;
 4663	int i, j;
 4664
 4665	if (env->log.level & BPF_LOG_LEVEL2) {
 4666		verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n",
 4667			st->curframe);
 4668	}
 4669
 4670	/* big hammer: mark all scalars precise in this path.
 4671	 * pop_stack may still get !precise scalars.
 4672	 * We also skip current state and go straight to first parent state,
 4673	 * because precision markings in current non-checkpointed state are
 4674	 * not needed. See why in the comment in __mark_chain_precision below.
 4675	 */
 4676	for (st = st->parent; st; st = st->parent) {
 4677		for (i = 0; i <= st->curframe; i++) {
 4678			func = st->frame[i];
 4679			for (j = 0; j < BPF_REG_FP; j++) {
 4680				reg = &func->regs[j];
 4681				if (reg->type != SCALAR_VALUE || reg->precise)
 4682					continue;
 4683				reg->precise = true;
 4684				if (env->log.level & BPF_LOG_LEVEL2) {
 4685					verbose(env, "force_precise: frame%d: forcing r%d to be precise\n",
 4686						i, j);
 4687				}
 4688			}
 4689			for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
 4690				if (!is_spilled_reg(&func->stack[j]))
 4691					continue;
 4692				reg = &func->stack[j].spilled_ptr;
 4693				if (reg->type != SCALAR_VALUE || reg->precise)
 4694					continue;
 4695				reg->precise = true;
 4696				if (env->log.level & BPF_LOG_LEVEL2) {
 4697					verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n",
 4698						i, -(j + 1) * 8);
 4699				}
 4700			}
 4701		}
 4702	}
 4703}
 4704
 4705static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
 4706{
 4707	struct bpf_func_state *func;
 4708	struct bpf_reg_state *reg;
 4709	int i, j;
 4710
 4711	for (i = 0; i <= st->curframe; i++) {
 4712		func = st->frame[i];
 4713		for (j = 0; j < BPF_REG_FP; j++) {
 4714			reg = &func->regs[j];
 4715			if (reg->type != SCALAR_VALUE)
 4716				continue;
 4717			reg->precise = false;
 4718		}
 4719		for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
 4720			if (!is_spilled_reg(&func->stack[j]))
 4721				continue;
 4722			reg = &func->stack[j].spilled_ptr;
 4723			if (reg->type != SCALAR_VALUE)
 4724				continue;
 4725			reg->precise = false;
 4726		}
 4727	}
 4728}
 4729
 4730/*
 4731 * __mark_chain_precision() backtracks BPF program instruction sequence and
 4732 * chain of verifier states making sure that register *regno* (if regno >= 0)
 4733 * and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked
 4734 * SCALARS, as well as any other registers and slots that contribute to
 4735 * a tracked state of given registers/stack slots, depending on specific BPF
 4736 * assembly instructions (see backtrack_insns() for exact instruction handling
 4737 * logic). This backtracking relies on recorded jmp_history and is able to
 4738 * traverse entire chain of parent states. This process ends only when all the
 4739 * necessary registers/slots and their transitive dependencies are marked as
 4740 * precise.
 4741 *
 4742 * One important and subtle aspect is that precise marks *do not matter* in
 4743 * the currently verified state (current state). It is important to understand
 4744 * why this is the case.
 4745 *
 4746 * First, note that current state is the state that is not yet "checkpointed",
 4747 * i.e., it is not yet put into env->explored_states, and it has no children
 4748 * states as well. It's ephemeral, and can end up either a) being discarded if
 4749 * compatible explored state is found at some point or BPF_EXIT instruction is
 4750 * reached or b) checkpointed and put into env->explored_states, branching out
 4751 * into one or more children states.
 4752 *
 4753 * In the former case, precise markings in current state are completely
 4754 * ignored by state comparison code (see regsafe() for details). Only
 4755 * checkpointed ("old") state precise markings are important, and if old
 4756 * state's register/slot is precise, regsafe() assumes current state's
 4757 * register/slot as precise and checks value ranges exactly and precisely. If
 4758 * states turn out to be compatible, current state's necessary precise
 4759 * markings and any required parent states' precise markings are enforced
 4760 * after the fact with propagate_precision() logic, after the fact. But it's
 4761 * important to realize that in this case, even after marking current state
 4762 * registers/slots as precise, we immediately discard current state. So what
 4763 * actually matters is any of the precise markings propagated into current
 4764 * state's parent states, which are always checkpointed (due to b) case above).
 4765 * As such, for scenario a) it doesn't matter if current state has precise
 4766 * markings set or not.
 4767 *
 4768 * Now, for the scenario b), checkpointing and forking into child(ren)
 4769 * state(s). Note that before current state gets to checkpointing step, any
 4770 * processed instruction always assumes precise SCALAR register/slot
 4771 * knowledge: if precise value or range is useful to prune jump branch, BPF
 4772 * verifier takes this opportunity enthusiastically. Similarly, when
 4773 * register's value is used to calculate offset or memory address, exact
 4774 * knowledge of SCALAR range is assumed, checked, and enforced. So, similar to
 4775 * what we mentioned above about state comparison ignoring precise markings
 4776 * during state comparison, BPF verifier ignores and also assumes precise
 4777 * markings *at will* during instruction verification process. But as verifier
 4778 * assumes precision, it also propagates any precision dependencies across
 4779 * parent states, which are not yet finalized, so can be further restricted
 4780 * based on new knowledge gained from restrictions enforced by their children
 4781 * states. This is so that once those parent states are finalized, i.e., when
 4782 * they have no more active children state, state comparison logic in
 4783 * is_state_visited() would enforce strict and precise SCALAR ranges, if
 4784 * required for correctness.
 4785 *
 4786 * To build a bit more intuition, note also that once a state is checkpointed,
 4787 * the path we took to get to that state is not important. This is crucial
 4788 * property for state pruning. When state is checkpointed and finalized at
 4789 * some instruction index, it can be correctly and safely used to "short
 4790 * circuit" any *compatible* state that reaches exactly the same instruction
 4791 * index. I.e., if we jumped to that instruction from a completely different
 4792 * code path than original finalized state was derived from, it doesn't
 4793 * matter, current state can be discarded because from that instruction
 4794 * forward having a compatible state will ensure we will safely reach the
 4795 * exit. States describe preconditions for further exploration, but completely
 4796 * forget the history of how we got here.
 4797 *
 4798 * This also means that even if we needed precise SCALAR range to get to
 4799 * finalized state, but from that point forward *that same* SCALAR register is
 4800 * never used in a precise context (i.e., it's precise value is not needed for
 4801 * correctness), it's correct and safe to mark such register as "imprecise"
 4802 * (i.e., precise marking set to false). This is what we rely on when we do
 4803 * not set precise marking in current state. If no child state requires
 4804 * precision for any given SCALAR register, it's safe to dictate that it can
 4805 * be imprecise. If any child state does require this register to be precise,
 4806 * we'll mark it precise later retroactively during precise markings
 4807 * propagation from child state to parent states.
 4808 *
 4809 * Skipping precise marking setting in current state is a mild version of
 4810 * relying on the above observation. But we can utilize this property even
 4811 * more aggressively by proactively forgetting any precise marking in the
 4812 * current state (which we inherited from the parent state), right before we
 4813 * checkpoint it and branch off into new child state. This is done by
 4814 * mark_all_scalars_imprecise() to hopefully get more permissive and generic
 4815 * finalized states which help in short circuiting more future states.
 4816 */
 4817static int __mark_chain_precision(struct bpf_verifier_env *env,
 4818				  struct bpf_verifier_state *starting_state,
 4819				  int regno,
 4820				  bool *changed)
 4821{
 4822	struct bpf_verifier_state *st = starting_state;
 4823	struct backtrack_state *bt = &env->bt;
 4824	int first_idx = st->first_insn_idx;
 4825	int last_idx = starting_state->insn_idx;
 4826	int subseq_idx = -1;
 4827	struct bpf_func_state *func;
 4828	bool tmp, skip_first = true;
 4829	struct bpf_reg_state *reg;
 4830	int i, fr, err;
 4831
 4832	if (!env->bpf_capable)
 4833		return 0;
 4834
 4835	changed = changed ?: &tmp;
 4836	/* set frame number from which we are starting to backtrack */
 4837	bt_init(bt, starting_state->curframe);
 4838
 4839	/* Do sanity checks against current state of register and/or stack
 4840	 * slot, but don't set precise flag in current state, as precision
 4841	 * tracking in the current state is unnecessary.
 4842	 */
 4843	func = st->frame[bt->frame];
 4844	if (regno >= 0) {
 4845		reg = &func->regs[regno];
 4846		if (reg->type != SCALAR_VALUE) {
 4847			verifier_bug(env, "backtracking misuse");
 4848			return -EFAULT;
 4849		}
 4850		bt_set_reg(bt, regno);
 4851	}
 4852
 4853	if (bt_empty(bt))
 4854		return 0;
 4855
 4856	for (;;) {
 4857		DECLARE_BITMAP(mask, 64);
 4858		u32 history = st->jmp_history_cnt;
 4859		struct bpf_jmp_history_entry *hist;
 4860
 4861		if (env->log.level & BPF_LOG_LEVEL2) {
 4862			verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n",
 4863				bt->frame, last_idx, first_idx, subseq_idx);
 4864		}
 4865
 4866		if (last_idx < 0) {
 4867			/* we are at the entry into subprog, which
 4868			 * is expected for global funcs, but only if
 4869			 * requested precise registers are R1-R5
 4870			 * (which are global func's input arguments)
 4871			 */
 4872			if (st->curframe == 0 &&
 4873			    st->frame[0]->subprogno > 0 &&
 4874			    st->frame[0]->callsite == BPF_MAIN_FUNC &&
 4875			    bt_stack_mask(bt) == 0 &&
 4876			    (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) {
 4877				bitmap_from_u64(mask, bt_reg_mask(bt));
 4878				for_each_set_bit(i, mask, 32) {
 4879					reg = &st->frame[0]->regs[i];
 4880					bt_clear_reg(bt, i);
 4881					if (reg->type == SCALAR_VALUE) {
 4882						reg->precise = true;
 4883						*changed = true;
 4884					}
 4885				}
 4886				return 0;
 4887			}
 4888
 4889			verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx",
 4890				     st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt));
 4891			return -EFAULT;
 4892		}
 4893
 4894		for (i = last_idx;;) {
 4895			if (skip_first) {
 4896				err = 0;
 4897				skip_first = false;
 4898			} else {
 4899				hist = get_jmp_hist_entry(st, history, i);
 4900				err = backtrack_insn(env, i, subseq_idx, hist, bt);
 4901			}
 4902			if (err == -ENOTSUPP) {
 4903				mark_all_scalars_precise(env, starting_state);
 4904				bt_reset(bt);
 4905				return 0;
 4906			} else if (err) {
 4907				return err;
 4908			}
 4909			if (bt_empty(bt))
 4910				/* Found assignment(s) into tracked register in this state.
 4911				 * Since this state is already marked, just return.
 4912				 * Nothing to be tracked further in the parent state.
 4913				 */
 4914				return 0;
 4915			subseq_idx = i;
 4916			i = get_prev_insn_idx(st, i, &history);
 4917			if (i == -ENOENT)
 4918				break;
 4919			if (i >= env->prog->len) {
 4920				/* This can happen if backtracking reached insn 0
 4921				 * and there are still reg_mask or stack_mask
 4922				 * to backtrack.
 4923				 * It means the backtracking missed the spot where
 4924				 * particular register was initialized with a constant.
 4925				 */
 4926				verifier_bug(env, "backtracking idx %d", i);
 4927				return -EFAULT;
 4928			}
 4929		}
 4930		st = st->parent;
 4931		if (!st)
 4932			break;
 4933
 4934		for (fr = bt->frame; fr >= 0; fr--) {
 4935			func = st->frame[fr];
 4936			bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr));
 4937			for_each_set_bit(i, mask, 32) {
 4938				reg = &func->regs[i];
 4939				if (reg->type != SCALAR_VALUE) {
 4940					bt_clear_frame_reg(bt, fr, i);
 4941					continue;
 4942				}
 4943				if (reg->precise) {
 4944					bt_clear_frame_reg(bt, fr, i);
 4945				} else {
 4946					reg->precise = true;
 4947					*changed = true;
 4948				}
 4949			}
 4950
 4951			bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr));
 4952			for_each_set_bit(i, mask, 64) {
 4953				if (verifier_bug_if(i >= func->allocated_stack / BPF_REG_SIZE,
 4954						    env, "stack slot %d, total slots %d",
 4955						    i, func->allocated_stack / BPF_REG_SIZE))
 4956					return -EFAULT;
 4957
 4958				if (!is_spilled_scalar_reg(&func->stack[i])) {
 4959					bt_clear_frame_slot(bt, fr, i);
 4960					continue;
 4961				}
 4962				reg = &func->stack[i].spilled_ptr;
 4963				if (reg->precise) {
 4964					bt_clear_frame_slot(bt, fr, i);
 4965				} else {
 4966					reg->precise = true;
 4967					*changed = true;
 4968				}
 4969			}
 4970			if (env->log.level & BPF_LOG_LEVEL2) {
 4971				fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
 4972					     bt_frame_reg_mask(bt, fr));
 4973				verbose(env, "mark_precise: frame%d: parent state regs=%s ",
 4974					fr, env->tmp_str_buf);
 4975				bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
 4976					       bt_frame_stack_mask(bt, fr));
 4977				verbose(env, "stack=%s: ", env->tmp_str_buf);
 4978				print_verifier_state(env, st, fr, true);
 4979			}
 4980		}
 4981
 4982		if (bt_empty(bt))
 4983			return 0;
 4984
 4985		subseq_idx = first_idx;
 4986		last_idx = st->last_insn_idx;
 4987		first_idx = st->first_insn_idx;
 4988	}
 4989
 4990	/* if we still have requested precise regs or slots, we missed
 4991	 * something (e.g., stack access through non-r10 register), so
 4992	 * fallback to marking all precise
 4993	 */
 4994	if (!bt_empty(bt)) {
 4995		mark_all_scalars_precise(env, starting_state);
 4996		bt_reset(bt);
 4997	}
 4998
 4999	return 0;
 5000}
 5001
 5002int mark_chain_precision(struct bpf_verifier_env *env, int regno)
 5003{
 5004	return __mark_chain_precision(env, env->cur_state, regno, NULL);
 5005}
 5006
 5007/* mark_chain_precision_batch() assumes that env->bt is set in the caller to
 5008 * desired reg and stack masks across all relevant frames
 5009 */
 5010static int mark_chain_precision_batch(struct bpf_verifier_env *env,
 5011				      struct bpf_verifier_state *starting_state)
 5012{
 5013	return __mark_chain_precision(env, starting_state, -1, NULL);
 5014}
 5015
 5016static bool is_spillable_regtype(enum bpf_reg_type type)
 5017{
 5018	switch (base_type(type)) {
 5019	case PTR_TO_MAP_VALUE:
 5020	case PTR_TO_STACK:
 5021	case PTR_TO_CTX:
 5022	case PTR_TO_PACKET:
 5023	case PTR_TO_PACKET_META:
 5024	case PTR_TO_PACKET_END:
 5025	case PTR_TO_FLOW_KEYS:
 5026	case CONST_PTR_TO_MAP:
 5027	case PTR_TO_SOCKET:
 5028	case PTR_TO_SOCK_COMMON:
 5029	case PTR_TO_TCP_SOCK:
 5030	case PTR_TO_XDP_SOCK:
 5031	case PTR_TO_BTF_ID:
 5032	case PTR_TO_BUF:
 5033	case PTR_TO_MEM:
 5034	case PTR_TO_FUNC:
 5035	case PTR_TO_MAP_KEY:
 5036	case PTR_TO_ARENA:
 5037		return true;
 5038	default:
 5039		return false;
 5040	}
 5041}
 5042
 5043/* Does this register contain a constant zero? */
 5044static bool register_is_null(struct bpf_reg_state *reg)
 5045{
 5046	return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0);
 5047}
 5048
 5049/* check if register is a constant scalar value */
 5050static bool is_reg_const(struct bpf_reg_state *reg, bool subreg32)
 5051{
 5052	return reg->type == SCALAR_VALUE &&
 5053	       tnum_is_const(subreg32 ? tnum_subreg(reg->var_off) : reg->var_off);
 5054}
 5055
 5056/* assuming is_reg_const() is true, return constant value of a register */
 5057static u64 reg_const_value(struct bpf_reg_state *reg, bool subreg32)
 5058{
 5059	return subreg32 ? tnum_subreg(reg->var_off).value : reg->var_off.value;
 5060}
 5061
 5062static bool __is_pointer_value(bool allow_ptr_leaks,
 5063			       const struct bpf_reg_state *reg)
 5064{
 5065	if (allow_ptr_leaks)
 5066		return false;
 5067
 5068	return reg->type != SCALAR_VALUE;
 5069}
 5070
 5071static void assign_scalar_id_before_mov(struct bpf_verifier_env *env,
 5072					struct bpf_reg_state *src_reg)
 5073{
 5074	if (src_reg->type != SCALAR_VALUE)
 5075		return;
 5076
 5077	if (src_reg->id & BPF_ADD_CONST) {
 5078		/*
 5079		 * The verifier is processing rX = rY insn and
 5080		 * rY->id has special linked register already.
 5081		 * Cleared it, since multiple rX += const are not supported.
 5082		 */
 5083		src_reg->id = 0;
 5084		src_reg->off = 0;
 5085	}
 5086
 5087	if (!src_reg->id && !tnum_is_const(src_reg->var_off))
 5088		/* Ensure that src_reg has a valid ID that will be copied to
 5089		 * dst_reg and then will be used by sync_linked_regs() to
 5090		 * propagate min/max range.
 5091		 */
 5092		src_reg->id = ++env->id_gen;
 5093}
 5094
 5095/* Copy src state preserving dst->parent and dst->live fields */
 5096static void copy_register_state(struct bpf_reg_state *dst, const struct bpf_reg_state *src)
 5097{
 5098	*dst = *src;
 5099}
 5100
 5101static void save_register_state(struct bpf_verifier_env *env,
 5102				struct bpf_func_state *state,
 5103				int spi, struct bpf_reg_state *reg,
 5104				int size)
 5105{
 5106	int i;
 5107
 5108	copy_register_state(&state->stack[spi].spilled_ptr, reg);
 5109
 5110	for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--)
 5111		state->stack[spi].slot_type[i - 1] = STACK_SPILL;
 5112
 5113	/* size < 8 bytes spill */
 5114	for (; i; i--)
 5115		mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]);
 5116}
 5117
 5118static bool is_bpf_st_mem(struct bpf_insn *insn)
 5119{
 5120	return BPF_CLASS(insn->code) == BPF_ST && BPF_MODE(insn->code) == BPF_MEM;
 5121}
 5122
 5123static int get_reg_width(struct bpf_reg_state *reg)
 5124{
 5125	return fls64(reg->umax_value);
 5126}
 5127
 5128/* See comment for mark_fastcall_pattern_for_call() */
 5129static void check_fastcall_stack_contract(struct bpf_verifier_env *env,
 5130					  struct bpf_func_state *state, int insn_idx, int off)
 5131{
 5132	struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];
 5133	struct bpf_insn_aux_data *aux = env->insn_aux_data;
 5134	int i;
 5135
 5136	if (subprog->fastcall_stack_off <= off || aux[insn_idx].fastcall_pattern)
 5137		return;
 5138	/* access to the region [max_stack_depth .. fastcall_stack_off)
 5139	 * from something that is not a part of the fastcall pattern,
 5140	 * disable fastcall rewrites for current subprogram by setting
 5141	 * fastcall_stack_off to a value smaller than any possible offset.
 5142	 */
 5143	subprog->fastcall_stack_off = S16_MIN;
 5144	/* reset fastcall aux flags within subprogram,
 5145	 * happens at most once per subprogram
 5146	 */
 5147	for (i = subprog->start; i < (subprog + 1)->start; ++i) {
 5148		aux[i].fastcall_spills_num = 0;
 5149		aux[i].fastcall_pattern = 0;
 5150	}
 5151}
 5152
 5153/* check_stack_{read,write}_fixed_off functions track spill/fill of registers,
 5154 * stack boundary and alignment are checked in check_mem_access()
 5155 */
 5156static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
 5157				       /* stack frame we're writing to */
 5158				       struct bpf_func_state *state,
 5159				       int off, int size, int value_regno,
 5160				       int insn_idx)
 5161{
 5162	struct bpf_func_state *cur; /* state of the current function */
 5163	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
 5164	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
 5165	struct bpf_reg_state *reg = NULL;
 5166	int insn_flags = insn_stack_access_flags(state->frameno, spi);
 5167
 5168	/* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0,
 5169	 * so it's aligned access and [off, off + size) are within stack limits
 5170	 */
 5171	if (!env->allow_ptr_leaks &&
 5172	    is_spilled_reg(&state->stack[spi]) &&
 5173	    !is_spilled_scalar_reg(&state->stack[spi]) &&
 5174	    size != BPF_REG_SIZE) {
 5175		verbose(env, "attempt to corrupt spilled pointer on stack\n");
 5176		return -EACCES;
 5177	}
 5178
 5179	cur = env->cur_state->frame[env->cur_state->curframe];
 5180	if (value_regno >= 0)
 5181		reg = &cur->regs[value_regno];
 5182	if (!env->bypass_spec_v4) {
 5183		bool sanitize = reg && is_spillable_regtype(reg->type);
 5184
 5185		for (i = 0; i < size; i++) {
 5186			u8 type = state->stack[spi].slot_type[i];
 5187
 5188			if (type != STACK_MISC && type != STACK_ZERO) {
 5189				sanitize = true;
 5190				break;
 5191			}
 5192		}
 5193
 5194		if (sanitize)
 5195			env->insn_aux_data[insn_idx].nospec_result = true;
 5196	}
 5197
 5198	err = destroy_if_dynptr_stack_slot(env, state, spi);
 5199	if (err)
 5200		return err;
 5201
 5202	if (!(off % BPF_REG_SIZE) && size == BPF_REG_SIZE) {
 5203		/* only mark the slot as written if all 8 bytes were written
 5204		 * otherwise read propagation may incorrectly stop too soon
 5205		 * when stack slots are partially written.
 5206		 * This heuristic means that read propagation will be
 5207		 * conservative, since it will add reg_live_read marks
 5208		 * to stack slots all the way to first state when programs
 5209		 * writes+reads less than 8 bytes
 5210		 */
 5211		bpf_mark_stack_write(env, state->frameno, BIT(spi));
 5212	}
 5213
 5214	check_fastcall_stack_contract(env, state, insn_idx, off);
 5215	mark_stack_slot_scratched(env, spi);
 5216	if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) {
 5217		bool reg_value_fits;
 5218
 5219		reg_value_fits = get_reg_width(reg) <= BITS_PER_BYTE * size;
 5220		/* Make sure that reg had an ID to build a relation on spill. */
 5221		if (reg_value_fits)
 5222			assign_scalar_id_before_mov(env, reg);
 5223		save_register_state(env, state, spi, reg, size);
 5224		/* Break the relation on a narrowing spill. */
 5225		if (!reg_value_fits)
 5226			state->stack[spi].spilled_ptr.id = 0;
 5227	} else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) &&
 5228		   env->bpf_capable) {
 5229		struct bpf_reg_state *tmp_reg = &env->fake_reg[0];
 5230
 5231		memset(tmp_reg, 0, sizeof(*tmp_reg));
 5232		__mark_reg_known(tmp_reg, insn->imm);
 5233		tmp_reg->type = SCALAR_VALUE;
 5234		save_register_state(env, state, spi, tmp_reg, size);
 5235	} else if (reg && is_spillable_regtype(reg->type)) {
 5236		/* register containing pointer is being spilled into stack */
 5237		if (size != BPF_REG_SIZE) {
 5238			verbose_linfo(env, insn_idx, "; ");
 5239			verbose(env, "invalid size of register spill\n");
 5240			return -EACCES;
 5241		}
 5242		if (state != cur && reg->type == PTR_TO_STACK) {
 5243			verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
 5244			return -EINVAL;
 5245		}
 5246		save_register_state(env, state, spi, reg, size);
 5247	} else {
 5248		u8 type = STACK_MISC;
 5249
 5250		/* regular write of data into stack destroys any spilled ptr */
 5251		state->stack[spi].spilled_ptr.type = NOT_INIT;
 5252		/* Mark slots as STACK_MISC if they belonged to spilled ptr/dynptr/iter. */
 5253		if (is_stack_slot_special(&state->stack[spi]))
 5254			for (i = 0; i < BPF_REG_SIZE; i++)
 5255				scrub_spilled_slot(&state->stack[spi].slot_type[i]);
 5256
 5257		/* when we zero initialize stack slots mark them as such */
 5258		if ((reg && register_is_null(reg)) ||
 5259		    (!reg && is_bpf_st_mem(insn) && insn->imm == 0)) {
 5260			/* STACK_ZERO case happened because register spill
 5261			 * wasn't properly aligned at the stack slot boundary,
 5262			 * so it's not a register spill anymore; force
 5263			 * originating register to be precise to make
 5264			 * STACK_ZERO correct for subsequent states
 5265			 */
 5266			err = mark_chain_precision(env, value_regno);
 5267			if (err)
 5268				return err;
 5269			type = STACK_ZERO;
 5270		}
 5271
 5272		/* Mark slots affected by this stack write. */
 5273		for (i = 0; i < size; i++)
 5274			state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = type;
 5275		insn_flags = 0; /* not a register spill */
 5276	}
 5277
 5278	if (insn_flags)
 5279		return push_jmp_history(env, env->cur_state, insn_flags, 0);
 5280	return 0;
 5281}
 5282
 5283/* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is
 5284 * known to contain a variable offset.
 5285 * This function checks whether the write is permitted and conservatively
 5286 * tracks the effects of the write, considering that each stack slot in the
 5287 * dynamic range is potentially written to.
 5288 *
 5289 * 'off' includes 'regno->off'.
 5290 * 'value_regno' can be -1, meaning that an unknown value is being written to
 5291 * the stack.
 5292 *
 5293 * Spilled pointers in range are not marked as written because we don't know
 5294 * what's going to be actually written. This means that read propagation for
 5295 * future reads cannot be terminated by this write.
 5296 *
 5297 * For privileged programs, uninitialized stack slots are considered
 5298 * initialized by this write (even though we don't know exactly what offsets
 5299 * are going to be written to). The idea is that we don't want the verifier to
 5300 * reject future reads that access slots written to through variable offsets.
 5301 */
 5302static int check_stack_write_var_off(struct bpf_verifier_env *env,
 5303				     /* func where register points to */
 5304				     struct bpf_func_state *state,
 5305				     int ptr_regno, int off, int size,
 5306				     int value_regno, int insn_idx)
 5307{
 5308	struct bpf_func_state *cur; /* state of the current function */
 5309	int min_off, max_off;
 5310	int i, err;
 5311	struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL;
 5312	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
 5313	bool writing_zero = false;
 5314	/* set if the fact that we're writing a zero is used to let any
 5315	 * stack slots remain STACK_ZERO
 5316	 */
 5317	bool zero_used = false;
 5318
 5319	cur = env->cur_state->frame[env->cur_state->curframe];
 5320	ptr_reg = &cur->regs[ptr_regno];
 5321	min_off = ptr_reg->smin_value + off;
 5322	max_off = ptr_reg->smax_value + off + size;
 5323	if (value_regno >= 0)
 5324		value_reg = &cur->regs[value_regno];
 5325	if ((value_reg && register_is_null(value_reg)) ||
 5326	    (!value_reg && is_bpf_st_mem(insn) && insn->imm == 0))
 5327		writing_zero = true;
 5328
 5329	for (i = min_off; i < max_off; i++) {
 5330		int spi;
 5331
 5332		spi = __get_spi(i);
 5333		err = destroy_if_dynptr_stack_slot(env, state, spi);
 5334		if (err)
 5335			return err;
 5336	}
 5337
 5338	check_fastcall_stack_contract(env, state, insn_idx, min_off);
 5339	/* Variable offset writes destroy any spilled pointers in range. */
 5340	for (i = min_off; i < max_off; i++) {
 5341		u8 new_type, *stype;
 5342		int slot, spi;
 5343
 5344		slot = -i - 1;
 5345		spi = slot / BPF_REG_SIZE;
 5346		stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
 5347		mark_stack_slot_scratched(env, spi);
 5348
 5349		if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) {
 5350			/* Reject the write if range we may write to has not
 5351			 * been initialized beforehand. If we didn't reject
 5352			 * here, the ptr status would be erased below (even
 5353			 * though not all slots are actually overwritten),
 5354			 * possibly opening the door to leaks.
 5355			 *
 5356			 * We do however catch STACK_INVALID case below, and
 5357			 * only allow reading possibly uninitialized memory
 5358			 * later for CAP_PERFMON, as the write may not happen to
 5359			 * that slot.
 5360			 */
 5361			verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d",
 5362				insn_idx, i);
 5363			return -EINVAL;
 5364		}
 5365
 5366		/* If writing_zero and the spi slot contains a spill of value 0,
 5367		 * maintain the spill type.
 5368		 */
 5369		if (writing_zero && *stype == STACK_SPILL &&
 5370		    is_spilled_scalar_reg(&state->stack[spi])) {
 5371			struct bpf_reg_state *spill_reg = &state->stack[spi].spilled_ptr;
 5372
 5373			if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) {
 5374				zero_used = true;
 5375				continue;
 5376			}
 5377		}
 5378
 5379		/* Erase all other spilled pointers. */
 5380		state->stack[spi].spilled_ptr.type = NOT_INIT;
 5381
 5382		/* Update the slot type. */
 5383		new_type = STACK_MISC;
 5384		if (writing_zero && *stype == STACK_ZERO) {
 5385			new_type = STACK_ZERO;
 5386			zero_used = true;
 5387		}
 5388		/* If the slot is STACK_INVALID, we check whether it's OK to
 5389		 * pretend that it will be initialized by this write. The slot
 5390		 * might not actually be written to, and so if we mark it as
 5391		 * initialized future reads might leak uninitialized memory.
 5392		 * For privileged programs, we will accept such reads to slots
 5393		 * that may or may not be written because, if we're reject
 5394		 * them, the error would be too confusing.
 5395		 */
 5396		if (*stype == STACK_INVALID && !env->allow_uninit_stack) {
 5397			verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d",
 5398					insn_idx, i);
 5399			return -EINVAL;
 5400		}
 5401		*stype = new_type;
 5402	}
 5403	if (zero_used) {
 5404		/* backtracking doesn't work for STACK_ZERO yet. */
 5405		err = mark_chain_precision(env, value_regno);
 5406		if (err)
 5407			return err;
 5408	}
 5409	return 0;
 5410}
 5411
 5412/* When register 'dst_regno' is assigned some values from stack[min_off,
 5413 * max_off), we set the register's type according to the types of the
 5414 * respective stack slots. If all the stack values are known to be zeros, then
 5415 * so is the destination reg. Otherwise, the register is considered to be
 5416 * SCALAR. This function does not deal with register filling; the caller must
 5417 * ensure that all spilled registers in the stack range have been marked as
 5418 * read.
 5419 */
 5420static void mark_reg_stack_read(struct bpf_verifier_env *env,
 5421				/* func where src register points to */
 5422				struct bpf_func_state *ptr_state,
 5423				int min_off, int max_off, int dst_regno)
 5424{
 5425	struct bpf_verifier_state *vstate = env->cur_state;
 5426	struct bpf_func_state *state = vstate->frame[vstate->curframe];
 5427	int i, slot, spi;
 5428	u8 *stype;
 5429	int zeros = 0;
 5430
 5431	for (i = min_off; i < max_off; i++) {
 5432		slot = -i - 1;
 5433		spi = slot / BPF_REG_SIZE;
 5434		mark_stack_slot_scratched(env, spi);
 5435		stype = ptr_state->stack[spi].slot_type;
 5436		if (stype[slot % BPF_REG_SIZE] != STACK_ZERO)
 5437			break;
 5438		zeros++;
 5439	}
 5440	if (zeros == max_off - min_off) {
 5441		/* Any access_size read into register is zero extended,
 5442		 * so the whole register == const_zero.
 5443		 */
 5444		__mark_reg_const_zero(env, &state->regs[dst_regno]);
 5445	} else {
 5446		/* have read misc data from the stack */
 5447		mark_reg_unknown(env, state->regs, dst_regno);
 5448	}
 5449}
 5450
 5451/* Read the stack at 'off' and put the results into the register indicated by
 5452 * 'dst_regno'. It handles reg filling if the addressed stack slot is a
 5453 * spilled reg.
 5454 *
 5455 * 'dst_regno' can be -1, meaning that the read value is not going to a
 5456 * register.
 5457 *
 5458 * The access is assumed to be within the current stack bounds.
 5459 */
 5460static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
 5461				      /* func where src register points to */
 5462				      struct bpf_func_state *reg_state,
 5463				      int off, int size, int dst_regno)
 5464{
 5465	struct bpf_verifier_state *vstate = env->cur_state;
 5466	struct bpf_func_state *state = vstate->frame[vstate->curframe];
 5467	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
 5468	struct bpf_reg_state *reg;
 5469	u8 *stype, type;
 5470	int insn_flags = insn_stack_access_flags(reg_state->frameno, spi);
 5471	int err;
 5472
 5473	stype = reg_state->stack[spi].slot_type;
 5474	reg = &reg_state->stack[spi].spilled_ptr;
 5475
 5476	mark_stack_slot_scratched(env, spi);
 5477	check_fastcall_stack_contract(env, state, env->insn_idx, off);
 5478	err = bpf_mark_stack_read(env, reg_state->frameno, env->insn_idx, BIT(spi));
 5479	if (err)
 5480		return err;
 5481
 5482	if (is_spilled_reg(&reg_state->stack[spi])) {
 5483		u8 spill_size = 1;
 5484
 5485		for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--)
 5486			spill_size++;
 5487
 5488		if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) {
 5489			if (reg->type != SCALAR_VALUE) {
 5490				verbose_linfo(env, env->insn_idx, "; ");
 5491				verbose(env, "invalid size of register fill\n");
 5492				return -EACCES;
 5493			}
 5494
 5495			if (dst_regno < 0)
 5496				return 0;
 5497
 5498			if (size <= spill_size &&
 5499			    bpf_stack_narrow_access_ok(off, size, spill_size)) {
 5500				/* The earlier check_reg_arg() has decided the
 5501				 * subreg_def for this insn.  Save it first.
 5502				 */
 5503				s32 subreg_def = state->regs[dst_regno].subreg_def;
 5504
 5505				copy_register_state(&state->regs[dst_regno], reg);
 5506				state->regs[dst_regno].subreg_def = subreg_def;
 5507
 5508				/* Break the relation on a narrowing fill.
 5509				 * coerce_reg_to_size will adjust the boundaries.
 5510				 */
 5511				if (get_reg_width(reg) > size * BITS_PER_BYTE)
 5512					state->regs[dst_regno].id = 0;
 5513			} else {
 5514				int spill_cnt = 0, zero_cnt = 0;
 5515
 5516				for (i = 0; i < size; i++) {
 5517					type = stype[(slot - i) % BPF_REG_SIZE];
 5518					if (type == STACK_SPILL) {
 5519						spill_cnt++;
 5520						continue;
 5521					}
 5522					if (type == STACK_MISC)
 5523						continue;
 5524					if (type == STACK_ZERO) {
 5525						zero_cnt++;
 5526						continue;
 5527					}
 5528					if (type == STACK_INVALID && env->allow_uninit_stack)
 5529						continue;
 5530					verbose(env, "invalid read from stack off %d+%d size %d\n",
 5531						off, i, size);
 5532					return -EACCES;
 5533				}
 5534
 5535				if (spill_cnt == size &&
 5536				    tnum_is_const(reg->var_off) && reg->var_off.value == 0) {
 5537					__mark_reg_const_zero(env, &state->regs[dst_regno]);
 5538					/* this IS register fill, so keep insn_flags */
 5539				} else if (zero_cnt == size) {
 5540					/* similarly to mark_reg_stack_read(), preserve zeroes */
 5541					__mark_reg_const_zero(env, &state->regs[dst_regno]);
 5542					insn_flags = 0; /* not restoring original register state */
 5543				} else {
 5544					mark_reg_unknown(env, state->regs, dst_regno);
 5545					insn_flags = 0; /* not restoring original register state */
 5546				}
 5547			}
 5548		} else if (dst_regno >= 0) {
 5549			/* restore register state from stack */
 5550			copy_register_state(&state->regs[dst_regno], reg);
 5551			/* mark reg as written since spilled pointer state likely
 5552			 * has its liveness marks cleared by is_state_visited()
 5553			 * which resets stack/reg liveness for state transitions
 5554			 */
 5555		} else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
 5556			/* If dst_regno==-1, the caller is asking us whether
 5557			 * it is acceptable to use this value as a SCALAR_VALUE
 5558			 * (e.g. for XADD).
 5559			 * We must not allow unprivileged callers to do that
 5560			 * with spilled pointers.
 5561			 */
 5562			verbose(env, "leaking pointer from stack off %d\n",
 5563				off);
 5564			return -EACCES;
 5565		}
 5566	} else {
 5567		for (i = 0; i < size; i++) {
 5568			type = stype[(slot - i) % BPF_REG_SIZE];
 5569			if (type == STACK_MISC)
 5570				continue;
 5571			if (type == STACK_ZERO)
 5572				continue;
 5573			if (type == STACK_INVALID && env->allow_uninit_stack)
 5574				continue;
 5575			verbose(env, "invalid read from stack off %d+%d size %d\n",
 5576				off, i, size);
 5577			return -EACCES;
 5578		}
 5579		if (dst_regno >= 0)
 5580			mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
 5581		insn_flags = 0; /* we are not restoring spilled register */
 5582	}
 5583	if (insn_flags)
 5584		return push_jmp_history(env, env->cur_state, insn_flags, 0);
 5585	return 0;
 5586}
 5587
 5588enum bpf_access_src {
 5589	ACCESS_DIRECT = 1,  /* the access is performed by an instruction */
 5590	ACCESS_HELPER = 2,  /* the access is performed by a helper */
 5591};
 5592
 5593static int check_stack_range_initialized(struct bpf_verifier_env *env,
 5594					 int regno, int off, int access_size,
 5595					 bool zero_size_allowed,
 5596					 enum bpf_access_type type,
 5597					 struct bpf_call_arg_meta *meta);
 5598
 5599static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
 5600{
 5601	return cur_regs(env) + regno;
 5602}
 5603
 5604/* Read the stack at 'ptr_regno + off' and put the result into the register
 5605 * 'dst_regno'.
 5606 * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'),
 5607 * but not its variable offset.
 5608 * 'size' is assumed to be <= reg size and the access is assumed to be aligned.
 5609 *
 5610 * As opposed to check_stack_read_fixed_off, this function doesn't deal with
 5611 * filling registers (i.e. reads of spilled register cannot be detected when
 5612 * the offset is not fixed). We conservatively mark 'dst_regno' as containing
 5613 * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable
 5614 * offset; for a fixed offset check_stack_read_fixed_off should be used
 5615 * instead.
 5616 */
 5617static int check_stack_read_var_off(struct bpf_verifier_env *env,
 5618				    int ptr_regno, int off, int size, int dst_regno)
 5619{
 5620	/* The state of the source register. */
 5621	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
 5622	struct bpf_func_state *ptr_state = func(env, reg);
 5623	int err;
 5624	int min_off, max_off;
 5625
 5626	/* Note that we pass a NULL meta, so raw access will not be permitted.
 5627	 */
 5628	err = check_stack_range_initialized(env, ptr_regno, off, size,
 5629					    false, BPF_READ, NULL);
 5630	if (err)
 5631		return err;
 5632
 5633	min_off = reg->smin_value + off;
 5634	max_off = reg->smax_value + off;
 5635	mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
 5636	check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off);
 5637	return 0;
 5638}
 5639
 5640/* check_stack_read dispatches to check_stack_read_fixed_off or
 5641 * check_stack_read_var_off.
 5642 *
 5643 * The caller must ensure that the offset falls within the allocated stack
 5644 * bounds.
 5645 *
 5646 * 'dst_regno' is a register which will receive the value from the stack. It
 5647 * can be -1, meaning that the read value is not going to a register.
 5648 */
 5649static int check_stack_read(struct bpf_verifier_env *env,
 5650			    int ptr_regno, int off, int size,
 5651			    int dst_regno)
 5652{
 5653	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
 5654	struct bpf_func_state *state = func(env, reg);
 5655	int err;
 5656	/* Some accesses are only permitted with a static offset. */
 5657	bool var_off = !tnum_is_const(reg->var_off);
 5658
 5659	/* The offset is required to be static when reads don't go to a
 5660	 * register, in order to not leak pointers (see
 5661	 * check_stack_read_fixed_off).
 5662	 */
 5663	if (dst_regno < 0 && var_off) {
 5664		char tn_buf[48];
 5665
 5666		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
 5667		verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n",
 5668			tn_buf, off, size);
 5669		return -EACCES;
 5670	}
 5671	/* Variable offset is prohibited for unprivileged mode for simplicity
 5672	 * since it requires corresponding support in Spectre masking for stack
 5673	 * ALU. See also retrieve_ptr_limit(). The check in
 5674	 * check_stack_access_for_ptr_arithmetic() called by
 5675	 * adjust_ptr_min_max_vals() prevents users from creating stack pointers
 5676	 * with variable offsets, therefore no check is required here. Further,
 5677	 * just checking it here would be insufficient as speculative stack
 5678	 * writes could still lead to unsafe speculative behaviour.
 5679	 */
 5680	if (!var_off) {
 5681		off += reg->var_off.value;
 5682		err = check_stack_read_fixed_off(env, state, off, size,
 5683						 dst_regno);
 5684	} else {
 5685		/* Variable offset stack reads need more conservative handling
 5686		 * than fixed offset ones. Note that dst_regno >= 0 on this
 5687		 * branch.
 5688		 */
 5689		err = check_stack_read_var_off(env, ptr_regno, off, size,
 5690					       dst_regno);
 5691	}
 5692	return err;
 5693}
 5694
 5695
 5696/* check_stack_write dispatches to check_stack_write_fixed_off or
 5697 * check_stack_write_var_off.
 5698 *
 5699 * 'ptr_regno' is the register used as a pointer into the stack.
 5700 * 'off' includes 'ptr_regno->off', but not its variable offset (if any).
 5701 * 'value_regno' is the register whose value we're writing to the stack. It can
 5702 * be -1, meaning that we're not writing from a register.
 5703 *
 5704 * The caller must ensure that the offset falls within the maximum stack size.
 5705 */
 5706static int check_stack_write(struct bpf_verifier_env *env,
 5707			     int ptr_regno, int off, int size,
 5708			     int value_regno, int insn_idx)
 5709{
 5710	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
 5711	struct bpf_func_state *state = func(env, reg);
 5712	int err;
 5713
 5714	if (tnum_is_const(reg->var_off)) {
 5715		off += reg->var_off.value;
 5716		err = check_stack_write_fixed_off(env, state, off, size,
 5717						  value_regno, insn_idx);
 5718	} else {
 5719		/* Variable offset stack reads need more conservative handling
 5720		 * than fixed offset ones.
 5721		 */
 5722		err = check_stack_write_var_off(env, state,
 5723						ptr_regno, off, size,
 5724						value_regno, insn_idx);
 5725	}
 5726	return err;
 5727}
 5728
 5729static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,
 5730				 int off, int size, enum bpf_access_type type)
 5731{
 5732	struct bpf_reg_state *regs = cur_regs(env);
 5733	struct bpf_map *map = regs[regno].map_ptr;
 5734	u32 cap = bpf_map_flags_to_cap(map);
 5735
 5736	if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) {
 5737		verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n",
 5738			map->value_size, off, size);
 5739		return -EACCES;
 5740	}
 5741
 5742	if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) {
 5743		verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n",
 5744			map->value_size, off, size);
 5745		return -EACCES;
 5746	}
 5747
 5748	return 0;
 5749}
 5750
 5751/* check read/write into memory region (e.g., map value, ringbuf sample, etc) */
 5752static int __check_mem_access(struct bpf_verifier_env *env, int regno,
 5753			      int off, int size, u32 mem_size,
 5754			      bool zero_size_allowed)
 5755{
 5756	bool size_ok = size > 0 || (size == 0 && zero_size_allowed);
 5757	struct bpf_reg_state *reg;
 5758
 5759	if (off >= 0 && size_ok && (u64)off + size <= mem_size)
 5760		return 0;
 5761
 5762	reg = &cur_regs(env)[regno];
 5763	switch (reg->type) {
 5764	case PTR_TO_MAP_KEY:
 5765		verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n",
 5766			mem_size, off, size);
 5767		break;
 5768	case PTR_TO_MAP_VALUE:
 5769		verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n",
 5770			mem_size, off, size);
 5771		break;
 5772	case PTR_TO_PACKET:
 5773	case PTR_TO_PACKET_META:
 5774	case PTR_TO_PACKET_END:
 5775		verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n",
 5776			off, size, regno, reg->id, off, mem_size);
 5777		break;
 5778	case PTR_TO_MEM:
 5779	default:
 5780		verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n",
 5781			mem_size, off, size);
 5782	}
 5783
 5784	return -EACCES;
 5785}
 5786
 5787/* check read/write into a memory region with possible variable offset */
 5788static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno,
 5789				   int off, int size, u32 mem_size,
 5790				   bool zero_size_allowed)
 5791{
 5792	struct bpf_verifier_state *vstate = env->cur_state;
 5793	struct bpf_func_state *state = vstate->frame[vstate->curframe];
 5794	struct bpf_reg_state *reg = &state->regs[regno];
 5795	int err;
 5796
 5797	/* We may have adjusted the register pointing to memory region, so we
 5798	 * need to try adding each of min_value and max_value to off
 5799	 * to make sure our theoretical access will be safe.
 5800	 *
 5801	 * The minimum value is only important with signed
 5802	 * comparisons where we can't assume the floor of a
 5803	 * value is 0.  If we are using signed variables for our
 5804	 * index'es we need to make sure that whatever we use
 5805	 * will have a set floor within our range.
 5806	 */
 5807	if (reg->smin_value < 0 &&
 5808	    (reg->smin_value == S64_MIN ||
 5809	     (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) ||
 5810	      reg->smin_value + off < 0)) {
 5811		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
 5812			regno);
 5813		return -EACCES;
 5814	}
 5815	err = __check_mem_access(env, regno, reg->smin_value + off, size,
 5816				 mem_size, zero_size_allowed);
 5817	if (err) {
 5818		verbose(env, "R%d min value is outside of the allowed memory range\n",
 5819			regno);
 5820		return err;
 5821	}
 5822
 5823	/* If we haven't set a max value then we need to bail since we can't be
 5824	 * sure we won't do bad things.
 5825	 * If reg->umax_value + off could overflow, treat that as unbounded too.
 5826	 */
 5827	if (reg->umax_value >= BPF_MAX_VAR_OFF) {
 5828		verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n",
 5829			regno);
 5830		return -EACCES;
 5831	}
 5832	err = __check_mem_access(env, regno, reg->umax_value + off, size,
 5833				 mem_size, zero_size_allowed);
 5834	if (err) {
 5835		verbose(env, "R%d max value is outside of the allowed memory range\n",
 5836			regno);
 5837		return err;
 5838	}
 5839
 5840	return 0;
 5841}
 5842
 5843static int __check_ptr_off_reg(struct bpf_verifier_env *env,
 5844			       const struct bpf_reg_state *reg, int regno,
 5845			       bool fixed_off_ok)
 5846{
 5847	/* Access to this pointer-typed register or passing it to a helper
 5848	 * is only allowed in its original, unmodified form.
 5849	 */
 5850
 5851	if (reg->off < 0) {
 5852		verbose(env, "negative offset %s ptr R%d off=%d disallowed\n",
 5853			reg_type_str(env, reg->type), regno, reg->off);
 5854		return -EACCES;
 5855	}
 5856
 5857	if (!fixed_off_ok && reg->off) {
 5858		verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n",
 5859			reg_type_str(env, reg->type), regno, reg->off);
 5860		return -EACCES;
 5861	}
 5862
 5863	if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
 5864		char tn_buf[48];
 5865
 5866		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
 5867		verbose(env, "variable %s access var_off=%s disallowed\n",
 5868			reg_type_str(env, reg->type), tn_buf);
 5869		return -EACCES;
 5870	}
 5871
 5872	return 0;
 5873}
 5874
 5875static int check_ptr_off_reg(struct bpf_verifier_env *env,
 5876		             const struct bpf_reg_state *reg, int regno)
 5877{
 5878	return __check_ptr_off_reg(env, reg, regno, false);
 5879}
 5880
 5881static int map_kptr_match_type(struct bpf_verifier_env *env,
 5882			       struct btf_field *kptr_field,
 5883			       struct bpf_reg_state *reg, u32 regno)
 5884{
 5885	const char *targ_name = btf_type_name(kptr_field->kptr.btf, kptr_field->kptr.btf_id);
 5886	int perm_flags;
 5887	const char *reg_name = "";
 5888
 5889	if (btf_is_kernel(reg->btf)) {
 5890		perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED | MEM_RCU;
 5891
 5892		/* Only unreferenced case accepts untrusted pointers */
 5893		if (kptr_field->type == BPF_KPTR_UNREF)
 5894			perm_flags |= PTR_UNTRUSTED;
 5895	} else {
 5896		perm_flags = PTR_MAYBE_NULL | MEM_ALLOC;
 5897		if (kptr_field->type == BPF_KPTR_PERCPU)
 5898			perm_flags |= MEM_PERCPU;
 5899	}
 5900
 5901	if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags))
 5902		goto bad_type;
 5903
 5904	/* We need to verify reg->type and reg->btf, before accessing reg->btf */
 5905	reg_name = btf_type_name(reg->btf, reg->btf_id);
 5906
 5907	/* For ref_ptr case, release function check should ensure we get one
 5908	 * referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the
 5909	 * normal store of unreferenced kptr, we must ensure var_off is zero.
 5910	 * Since ref_ptr cannot be accessed directly by BPF insns, checks for
 5911	 * reg->off and reg->ref_obj_id are not needed here.
 5912	 */
 5913	if (__check_ptr_off_reg(env, reg, regno, true))
 5914		return -EACCES;
 5915
 5916	/* A full type match is needed, as BTF can be vmlinux, module or prog BTF, and
 5917	 * we also need to take into account the reg->off.
 5918	 *
 5919	 * We want to support cases like:
 5920	 *
 5921	 * struct foo {
 5922	 *         struct bar br;
 5923	 *         struct baz bz;
 5924	 * };
 5925	 *
 5926	 * struct foo *v;
 5927	 * v = func();	      // PTR_TO_BTF_ID
 5928	 * val->foo = v;      // reg->off is zero, btf and btf_id match type
 5929	 * val->bar = &v->br; // reg->off is still zero, but we need to retry with
 5930	 *                    // first member type of struct after comparison fails
 5931	 * val->baz = &v->bz; // reg->off is non-zero, so struct needs to be walked
 5932	 *                    // to match type
 5933	 *
 5934	 * In the kptr_ref case, check_func_arg_reg_off already ensures reg->off
 5935	 * is zero. We must also ensure that btf_struct_ids_match does not walk
 5936	 * the struct to match type against first member of struct, i.e. reject
 5937	 * second case from above. Hence, when type is BPF_KPTR_REF, we set
 5938	 * strict mode to true for type match.
 5939	 */
 5940	if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
 5941				  kptr_field->kptr.btf, kptr_field->kptr.btf_id,
 5942				  kptr_field->type != BPF_KPTR_UNREF))
 5943		goto bad_type;
 5944	return 0;
 5945bad_type:
 5946	verbose(env, "invalid kptr access, R%d type=%s%s ", regno,
 5947		reg_type_str(env, reg->type), reg_name);
 5948	verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name);
 5949	if (kptr_field->type == BPF_KPTR_UNREF)
 5950		verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED),
 5951			targ_name);
 5952	else
 5953		verbose(env, "\n");
 5954	return -EINVAL;
 5955}
 5956
 5957static bool in_sleepable(struct bpf_verifier_env *env)
 5958{
 5959	return env->cur_state->in_sleepable;
 5960}
 5961
 5962/* The non-sleepable programs and sleepable programs with explicit bpf_rcu_read_lock()
 5963 * can dereference RCU protected pointers and result is PTR_TRUSTED.
 5964 */
 5965static bool in_rcu_cs(struct bpf_verifier_env *env)
 5966{
 5967	return env->cur_state->active_rcu_locks ||
 5968	       env->cur_state->active_locks ||
 5969	       !in_sleepable(env);
 5970}
 5971
 5972/* Once GCC supports btf_type_tag the following mechanism will be replaced with tag check */
 5973BTF_SET_START(rcu_protected_types)
 5974#ifdef CONFIG_NET
 5975BTF_ID(struct, prog_test_ref_kfunc)
 5976#endif
 5977#ifdef CONFIG_CGROUPS
 5978BTF_ID(struct, cgroup)
 5979#endif
 5980#ifdef CONFIG_BPF_JIT
 5981BTF_ID(struct, bpf_cpumask)
 5982#endif
 5983BTF_ID(struct, task_struct)
 5984#ifdef CONFIG_CRYPTO
 5985BTF_ID(struct, bpf_crypto_ctx)
 5986#endif
 5987BTF_SET_END(rcu_protected_types)
 5988
 5989static bool rcu_protected_object(const struct btf *btf, u32 btf_id)
 5990{
 5991	if (!btf_is_kernel(btf))
 5992		return true;
 5993	return btf_id_set_contains(&rcu_protected_types, btf_id);
 5994}
 5995
 5996static struct btf_record *kptr_pointee_btf_record(struct btf_field *kptr_field)
 5997{
 5998	struct btf_struct_meta *meta;
 5999
 6000	if (btf_is_kernel(kptr_field->kptr.btf))
 6001		return NULL;
 6002
 6003	meta = btf_find_struct_meta(kptr_field->kptr.btf,
 6004				    kptr_field->kptr.btf_id);
 6005
 6006	return meta ? meta->record : NULL;
 6007}
 6008
 6009static bool rcu_safe_kptr(const struct btf_field *field)
 6010{
 6011	const struct btf_field_kptr *kptr = &field->kptr;
 6012
 6013	return field->type == BPF_KPTR_PERCPU ||
 6014	       (field->type == BPF_KPTR_REF && rcu_protected_object(kptr->btf, kptr->btf_id));
 6015}
 6016
 6017static u32 btf_ld_kptr_type(struct bpf_verifier_env *env, struct btf_field *kptr_field)
 6018{
 6019	struct btf_record *rec;
 6020	u32 ret;
 6021
 6022	ret = PTR_MAYBE_NULL;
 6023	if (rcu_safe_kptr(kptr_field) && in_rcu_cs(env)) {
 6024		ret |= MEM_RCU;
 6025		if (kptr_field->type == BPF_KPTR_PERCPU)
 6026			ret |= MEM_PERCPU;
 6027		else if (!btf_is_kernel(kptr_field->kptr.btf))
 6028			ret |= MEM_ALLOC;
 6029
 6030		rec = kptr_pointee_btf_record(kptr_field);
 6031		if (rec && btf_record_has_field(rec, BPF_GRAPH_NODE))
 6032			ret |= NON_OWN_REF;
 6033	} else {
 6034		ret |= PTR_UNTRUSTED;
 6035	}
 6036
 6037	return ret;
 6038}
 6039
 6040static int mark_uptr_ld_reg(struct bpf_verifier_env *env, u32 regno,
 6041			    struct btf_field *field)
 6042{
 6043	struct bpf_reg_state *reg;
 6044	const struct btf_type *t;
 6045
 6046	t = btf_type_by_id(field->kptr.btf, field->kptr.btf_id);
 6047	mark_reg_known_zero(env, cur_regs(env), regno);
 6048	reg = reg_state(env, regno);
 6049	reg->type = PTR_TO_MEM | PTR_MAYBE_NULL;
 6050	reg->mem_size = t->size;
 6051	reg->id = ++env->id_gen;
 6052
 6053	return 0;
 6054}
 6055
 6056static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno,
 6057				 int value_regno, int insn_idx,
 6058				 struct btf_field *kptr_field)
 6059{
 6060	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
 6061	int class = BPF_CLASS(insn->code);
 6062	struct bpf_reg_state *val_reg;
 6063	int ret;
 6064
 6065	/* Things we already checked for in check_map_access and caller:
 6066	 *  - Reject cases where variable offset may touch kptr
 6067	 *  - size of access (must be BPF_DW)
 6068	 *  - tnum_is_const(reg->var_off)
 6069	 *  - kptr_field->offset == off + reg->var_off.value
 6070	 */
 6071	/* Only BPF_[LDX,STX,ST] | BPF_MEM | BPF_DW is supported */
 6072	if (BPF_MODE(insn->code) != BPF_MEM) {
 6073		verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n");
 6074		return -EACCES;
 6075	}
 6076
 6077	/* We only allow loading referenced kptr, since it will be marked as
 6078	 * untrusted, similar to unreferenced kptr.
 6079	 */
 6080	if (class != BPF_LDX &&
 6081	    (kptr_field->type == BPF_KPTR_REF || kptr_field->type == BPF_KPTR_PERCPU)) {
 6082		verbose(env, "store to referenced kptr disallowed\n");
 6083		return -EACCES;
 6084	}
 6085	if (class != BPF_LDX && kptr_field->type == BPF_UPTR) {
 6086		verbose(env, "store to uptr disallowed\n");
 6087		return -EACCES;
 6088	}
 6089
 6090	if (class == BPF_LDX) {
 6091		if (kptr_field->type == BPF_UPTR)
 6092			return mark_uptr_ld_reg(env, value_regno, kptr_field);
 6093
 6094		/* We can simply mark the value_regno receiving the pointer
 6095		 * value from map as PTR_TO_BTF_ID, with the correct type.
 6096		 */
 6097		ret = mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID,
 6098				      kptr_field->kptr.btf, kptr_field->kptr.btf_id,
 6099				      btf_ld_kptr_type(env, kptr_field));
 6100		if (ret < 0)
 6101			return ret;
 6102	} else if (class == BPF_STX) {
 6103		val_reg = reg_state(env, value_regno);
 6104		if (!register_is_null(val_reg) &&
 6105		    map_kptr_match_type(env, kptr_field, val_reg, value_regno))
 6106			return -EACCES;
 6107	} else if (class == BPF_ST) {
 6108		if (insn->imm) {
 6109			verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n",
 6110				kptr_field->offset);
 6111			return -EACCES;
 6112		}
 6113	} else {
 6114		verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n");
 6115		return -EACCES;
 6116	}
 6117	return 0;
 6118}
 6119
 6120/*
 6121 * Return the size of the memory region accessible from a pointer to map value.
 6122 * For INSN_ARRAY maps whole bpf_insn_array->ips array is accessible.
 6123 */
 6124static u32 map_mem_size(const struct bpf_map *map)
 6125{
 6126	if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY)
 6127		return map->max_entries * sizeof(long);
 6128
 6129	return map->value_size;
 6130}
 6131
 6132/* check read/write into a map element with possible variable offset */
 6133static int check_map_access(struct bpf_verifier_env *env, u32 regno,
 6134			    int off, int size, bool zero_size_allowed,
 6135			    enum bpf_access_src src)
 6136{
 6137	struct bpf_verifier_state *vstate = env->cur_state;
 6138	struct bpf_func_state *state = vstate->frame[vstate->curframe];
 6139	struct bpf_reg_state *reg = &state->regs[regno];
 6140	struct bpf_map *map = reg->map_ptr;
 6141	u32 mem_size = map_mem_size(map);
 6142	struct btf_record *rec;
 6143	int err, i;
 6144
 6145	err = check_mem_region_access(env, regno, off, size, mem_size, zero_size_allowed);
 6146	if (err)
 6147		return err;
 6148
 6149	if (IS_ERR_OR_NULL(map->record))
 6150		return 0;
 6151	rec = map->record;
 6152	for (i = 0; i < rec->cnt; i++) {
 6153		struct btf_field *field = &rec->fields[i];
 6154		u32 p = field->offset;
 6155
 6156		/* If any part of a field  can be touched by load/store, reject
 6157		 * this program. To check that [x1, x2) overlaps with [y1, y2),
 6158		 * it is sufficient to check x1 < y2 && y1 < x2.
 6159		 */
 6160		if (reg->smin_value + off < p + field->size &&
 6161		    p < reg->umax_value + off + size) {
 6162			switch (field->type) {
 6163			case BPF_KPTR_UNREF:
 6164			case BPF_KPTR_REF:
 6165			case BPF_KPTR_PERCPU:
 6166			case BPF_UPTR:
 6167				if (src != ACCESS_DIRECT) {
 6168					verbose(env, "%s cannot be accessed indirectly by helper\n",
 6169						btf_field_type_name(field->type));
 6170					return -EACCES;
 6171				}
 6172				if (!tnum_is_const(reg->var_off)) {
 6173					verbose(env, "%s access cannot have variable offset\n",
 6174						btf_field_type_name(field->type));
 6175					return -EACCES;
 6176				}
 6177				if (p != off + reg->var_off.value) {
 6178					verbose(env, "%s access misaligned expected=%u off=%llu\n",
 6179						btf_field_type_name(field->type),
 6180						p, off + reg->var_off.value);
 6181					return -EACCES;
 6182				}
 6183				if (size != bpf_size_to_bytes(BPF_DW)) {
 6184					verbose(env, "%s access size must be BPF_DW\n",
 6185						btf_field_type_name(field->type));
 6186					return -EACCES;
 6187				}
 6188				break;
 6189			default:
 6190				verbose(env, "%s cannot be accessed directly by load/store\n",
 6191					btf_field_type_name(field->type));
 6192				return -EACCES;
 6193			}
 6194		}
 6195	}
 6196	return 0;
 6197}
 6198
 6199#define MAX_PACKET_OFF 0xffff
 6200
 6201static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
 6202				       const struct bpf_call_arg_meta *meta,
 6203				       enum bpf_access_type t)
 6204{
 6205	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
 6206
 6207	switch (prog_type) {
 6208	/* Program types only with direct read access go here! */
 6209	case BPF_PROG_TYPE_LWT_IN:
 6210	case BPF_PROG_TYPE_LWT_OUT:
 6211	case BPF_PROG_TYPE_LWT_SEG6LOCAL:
 6212	case BPF_PROG_TYPE_SK_REUSEPORT:
 6213	case BPF_PROG_TYPE_FLOW_DISSECTOR:
 6214	case BPF_PROG_TYPE_CGROUP_SKB:
 6215		if (t == BPF_WRITE)
 6216			return false;
 6217		fallthrough;
 6218
 6219	/* Program types with direct read + write access go here! */
 6220	case BPF_PROG_TYPE_SCHED_CLS:
 6221	case BPF_PROG_TYPE_SCHED_ACT:
 6222	case BPF_PROG_TYPE_XDP:
 6223	case BPF_PROG_TYPE_LWT_XMIT:
 6224	case BPF_PROG_TYPE_SK_SKB:
 6225	case BPF_PROG_TYPE_SK_MSG:
 6226		if (meta)
 6227			return meta->pkt_access;
 6228
 6229		env->seen_direct_write = true;
 6230		return true;
 6231
 6232	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
 6233		if (t == BPF_WRITE)
 6234			env->seen_direct_write = true;
 6235
 6236		return true;
 6237
 6238	default:
 6239		return false;
 6240	}
 6241}
 6242
 6243static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,
 6244			       int size, bool zero_size_allowed)
 6245{
 6246	struct bpf_reg_state *regs = cur_regs(env);
 6247	struct bpf_reg_state *reg = &regs[regno];
 6248	int err;
 6249
 6250	/* We may have added a variable offset to the packet pointer; but any
 6251	 * reg->range we have comes after that.  We are only checking the fixed
 6252	 * offset.
 6253	 */
 6254
 6255	/* We don't allow negative numbers, because we aren't tracking enough
 6256	 * detail to prove they're safe.
 6257	 */
 6258	if (reg->smin_value < 0) {
 6259		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
 6260			regno);
 6261		return -EACCES;
 6262	}
 6263
 6264	err = reg->range < 0 ? -EINVAL :
 6265	      __check_mem_access(env, regno, off, size, reg->range,
 6266				 zero_size_allowed);
 6267	if (err) {
 6268		verbose(env, "R%d offset is outside of the packet\n", regno);
 6269		return err;
 6270	}
 6271
 6272	/* __check_mem_access has made sure "off + size - 1" is within u16.
 6273	 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff,
 6274	 * otherwise find_good_pkt_pointers would have refused to set range info
 6275	 * that __check_mem_access would have rejected this pkt access.
 6276	 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32.
 6277	 */
 6278	env->prog->aux->max_pkt_offset =
 6279		max_t(u32, env->prog->aux->max_pkt_offset,
 6280		      off + reg->umax_value + size - 1);
 6281
 6282	return err;
 6283}
 6284
 6285/* check access to 'struct bpf_context' fields.  Supports fixed offsets only */
 6286static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
 6287			    enum bpf_access_type t, struct bpf_insn_access_aux *info)
 6288{
 6289	if (env->ops->is_valid_access &&
 6290	    env->ops->is_valid_access(off, size, t, env->prog, info)) {
 6291		/* A non zero info.ctx_field_size indicates that this field is a
 6292		 * candidate for later verifier transformation to load the whole
 6293		 * field and then apply a mask when accessed with a narrower
 6294		 * access than actual ctx access size. A zero info.ctx_field_size
 6295		 * will only allow for whole field access and rejects any other
 6296		 * type of narrower access.
 6297		 */
 6298		if (base_type(info->reg_type) == PTR_TO_BTF_ID) {
 6299			if (info->ref_obj_id &&
 6300			    !find_reference_state(env->cur_state, info->ref_obj_id)) {
 6301				verbose(env, "invalid bpf_context access off=%d. Reference may already be released\n",
 6302					off);
 6303				return -EACCES;
 6304			}
 6305		} else {
 6306			env->insn_aux_data[insn_idx].ctx_field_size = info->ctx_field_size;
 6307		}
 6308		/* remember the offset of last byte accessed in ctx */
 6309		if (env->prog->aux->max_ctx_offset < off + size)
 6310			env->prog->aux->max_ctx_offset = off + size;
 6311		return 0;
 6312	}
 6313
 6314	verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);
 6315	return -EACCES;
 6316}
 6317
 6318static int check_flow_keys_access(struct bpf_verifier_env *env, int off,
 6319				  int size)
 6320{
 6321	if (size < 0 || off < 0 ||
 6322	    (u64)off + size > sizeof(struct bpf_flow_keys)) {
 6323		verbose(env, "invalid access to flow keys off=%d size=%d\n",
 6324			off, size);
 6325		return -EACCES;
 6326	}
 6327	return 0;
 6328}
 6329
 6330static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
 6331			     u32 regno, int off, int size,
 6332			     enum bpf_access_type t)
 6333{
 6334	struct bpf_reg_state *regs = cur_regs(env);
 6335	struct bpf_reg_state *reg = &regs[regno];
 6336	struct bpf_insn_access_aux info = {};
 6337	bool valid;
 6338
 6339	if (reg->smin_value < 0) {
 6340		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
 6341			regno);
 6342		return -EACCES;
 6343	}
 6344
 6345	switch (reg->type) {
 6346	case PTR_TO_SOCK_COMMON:
 6347		valid = bpf_sock_common_is_valid_access(off, size, t, &info);
 6348		break;
 6349	case PTR_TO_SOCKET:
 6350		valid = bpf_sock_is_valid_access(off, size, t, &info);
 6351		break;
 6352	case PTR_TO_TCP_SOCK:
 6353		valid = bpf_tcp_sock_is_valid_access(off, size, t, &info);
 6354		break;
 6355	case PTR_TO_XDP_SOCK:
 6356		valid = bpf_xdp_sock_is_valid_access(off, size, t, &info);
 6357		break;
 6358	default:
 6359		valid = false;
 6360	}
 6361
 6362
 6363	if (valid) {
 6364		env->insn_aux_data[insn_idx].ctx_field_size =
 6365			info.ctx_field_size;
 6366		return 0;
 6367	}
 6368
 6369	verbose(env, "R%d invalid %s access off=%d size=%d\n",
 6370		regno, reg_type_str(env, reg->type), off, size);
 6371
 6372	return -EACCES;
 6373}
 6374
 6375static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
 6376{
 6377	return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
 6378}
 6379
 6380static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)
 6381{
 6382	const struct bpf_reg_state *reg = reg_state(env, regno);
 6383
 6384	return reg->type == PTR_TO_CTX;
 6385}
 6386
 6387static bool is_sk_reg(struct bpf_verifier_env *env, int regno)
 6388{
 6389	const struct bpf_reg_state *reg = reg_state(env, regno);
 6390
 6391	return type_is_sk_pointer(reg->type);
 6392}
 6393
 6394static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)
 6395{
 6396	const struct bpf_reg_state *reg = reg_state(env, regno);
 6397
 6398	return type_is_pkt_pointer(reg->type);
 6399}
 6400
 6401static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)
 6402{
 6403	const struct bpf_reg_state *reg = reg_state(env, regno);
 6404
 6405	/* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */
 6406	return reg->type == PTR_TO_FLOW_KEYS;
 6407}
 6408
 6409static bool is_arena_reg(struct bpf_verifier_env *env, int regno)
 6410{
 6411	const struct bpf_reg_state *reg = reg_state(env, regno);
 6412
 6413	return reg->type == PTR_TO_ARENA;
 6414}
 6415
 6416/* Return false if @regno contains a pointer whose type isn't supported for
 6417 * atomic instruction @insn.
 6418 */
 6419static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno,
 6420			       struct bpf_insn *insn)
 6421{
 6422	if (is_ctx_reg(env, regno))
 6423		return false;
 6424	if (is_pkt_reg(env, regno))
 6425		return false;
 6426	if (is_flow_key_reg(env, regno))
 6427		return false;
 6428	if (is_sk_reg(env, regno))
 6429		return false;
 6430	if (is_arena_reg(env, regno))
 6431		return bpf_jit_supports_insn(insn, true);
 6432
 6433	return true;
 6434}
 6435
 6436static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
 6437#ifdef CONFIG_NET
 6438	[PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
 6439	[PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
 6440	[PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP],
 6441#endif
 6442	[CONST_PTR_TO_MAP] = btf_bpf_map_id,
 6443};
 6444
 6445static bool is_trusted_reg(const struct bpf_reg_state *reg)
 6446{
 6447	/* A referenced register is always trusted. */
 6448	if (reg->ref_obj_id)
 6449		return true;
 6450
 6451	/* Types listed in the reg2btf_ids are always trusted */
 6452	if (reg2btf_ids[base_type(reg->type)] &&
 6453	    !bpf_type_has_unsafe_modifiers(reg->type))
 6454		return true;
 6455
 6456	/* If a register is not referenced, it is trusted if it has the
 6457	 * MEM_ALLOC or PTR_TRUSTED type modifiers, and no others. Some of the
 6458	 * other type modifiers may be safe, but we elect to take an opt-in
 6459	 * approach here as some (e.g. PTR_UNTRUSTED and PTR_MAYBE_NULL) are
 6460	 * not.
 6461	 *
 6462	 * Eventually, we should make PTR_TRUSTED the single source of truth
 6463	 * for whether a register is trusted.
 6464	 */
 6465	return type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS &&
 6466	       !bpf_type_has_unsafe_modifiers(reg->type);
 6467}
 6468
 6469static bool is_rcu_reg(const struct bpf_reg_state *reg)
 6470{
 6471	return reg->type & MEM_RCU;
 6472}
 6473
 6474static void clear_trusted_flags(enum bpf_type_flag *flag)
 6475{
 6476	*flag &= ~(BPF_REG_TRUSTED_MODIFIERS | MEM_RCU);
 6477}
 6478
 6479static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,
 6480				   const struct bpf_reg_state *reg,
 6481				   int off, int size, bool strict)
 6482{
 6483	struct tnum reg_off;
 6484	int ip_align;
 6485
 6486	/* Byte size accesses are always allowed. */
 6487	if (!strict || size == 1)
 6488		return 0;
 6489
 6490	/* For platforms that do not have a Kconfig enabling
 6491	 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of
 6492	 * NET_IP_ALIGN is universally set to '2'.  And on platforms
 6493	 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get
 6494	 * to this code only in strict mode where we want to emulate
 6495	 * the NET_IP_ALIGN==2 checking.  Therefore use an
 6496	 * unconditional IP align value of '2'.
 6497	 */
 6498	ip_align = 2;
 6499
 6500	reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off));
 6501	if (!tnum_is_aligned(reg_off, size)) {
 6502		char tn_buf[48];
 6503
 6504		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
 6505		verbose(env,
 6506			"misaligned packet access off %d+%s+%d+%d size %d\n",
 6507			ip_align, tn_buf, reg->off, off, size);
 6508		return -EACCES;
 6509	}
 6510
 6511	return 0;
 6512}
 6513
 6514static int check_generic_ptr_alignment(struct bpf_verifier_env *env,
 6515				       const struct bpf_reg_state *reg,
 6516				       const char *pointer_desc,
 6517				       int off, int size, bool strict)
 6518{
 6519	struct tnum reg_off;
 6520
 6521	/* Byte size accesses are always allowed. */
 6522	if (!strict || size == 1)
 6523		return 0;
 6524
 6525	reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off));
 6526	if (!tnum_is_aligned(reg_off, size)) {
 6527		char tn_buf[48];
 6528
 6529		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
 6530		verbose(env, "misaligned %saccess off %s+%d+%d size %d\n",
 6531			pointer_desc, tn_buf, reg->off, off, size);
 6532		return -EACCES;
 6533	}
 6534
 6535	return 0;
 6536}
 6537
 6538static int check_ptr_alignment(struct bpf_verifier_env *env,
 6539			       const struct bpf_reg_state *reg, int off,
 6540			       int size, bool strict_alignment_once)
 6541{
 6542	bool strict = env->strict_alignment || strict_alignment_once;
 6543	const char *pointer_desc = "";
 6544
 6545	switch (reg->type) {
 6546	case PTR_TO_PACKET:
 6547	case PTR_TO_PACKET_META:
 6548		/* Special case, because of NET_IP_ALIGN. Given metadata sits
 6549		 * right in front, treat it the very same way.
 6550		 */
 6551		return check_pkt_ptr_alignment(env, reg, off, size, strict);
 6552	case PTR_TO_FLOW_KEYS:
 6553		pointer_desc = "flow keys ";
 6554		break;
 6555	case PTR_TO_MAP_KEY:
 6556		pointer_desc = "key ";
 6557		break;
 6558	case PTR_TO_MAP_VALUE:
 6559		pointer_desc = "value ";
 6560		if (reg->map_ptr->map_type == BPF_MAP_TYPE_INSN_ARRAY)
 6561			strict = true;
 6562		break;
 6563	case PTR_TO_CTX:
 6564		pointer_desc = "context ";
 6565		break;
 6566	case PTR_TO_STACK:
 6567		pointer_desc = "stack ";
 6568		/* The stack spill tracking logic in check_stack_write_fixed_off()
 6569		 * and check_stack_read_fixed_off() relies on stack accesses being
 6570		 * aligned.
 6571		 */
 6572		strict = true;
 6573		break;
 6574	case PTR_TO_SOCKET:
 6575		pointer_desc = "sock ";
 6576		break;
 6577	case PTR_TO_SOCK_COMMON:
 6578		pointer_desc = "sock_common ";
 6579		break;
 6580	case PTR_TO_TCP_SOCK:
 6581		pointer_desc = "tcp_sock ";
 6582		break;
 6583	case PTR_TO_XDP_SOCK:
 6584		pointer_desc = "xdp_sock ";
 6585		break;
 6586	case PTR_TO_ARENA:
 6587		return 0;
 6588	default:
 6589		break;
 6590	}
 6591	return check_generic_ptr_alignment(env, reg, pointer_desc, off, size,
 6592					   strict);
 6593}
 6594
 6595static enum priv_stack_mode bpf_enable_priv_stack(struct bpf_prog *prog)
 6596{
 6597	if (!bpf_jit_supports_private_stack())
 6598		return NO_PRIV_STACK;
 6599
 6600	/* bpf_prog_check_recur() checks all prog types that use bpf trampoline
 6601	 * while kprobe/tp/perf_event/raw_tp don't use trampoline hence checked
 6602	 * explicitly.
 6603	 */
 6604	switch (prog->type) {
 6605	case BPF_PROG_TYPE_KPROBE:
 6606	case BPF_PROG_TYPE_TRACEPOINT:
 6607	case BPF_PROG_TYPE_PERF_EVENT:
 6608	case BPF_PROG_TYPE_RAW_TRACEPOINT:
 6609		return PRIV_STACK_ADAPTIVE;
 6610	case BPF_PROG_TYPE_TRACING:
 6611	case BPF_PROG_TYPE_LSM:
 6612	case BPF_PROG_TYPE_STRUCT_OPS:
 6613		if (prog->aux->priv_stack_requested || bpf_prog_check_recur(prog))
 6614			return PRIV_STACK_ADAPTIVE;
 6615		fallthrough;
 6616	default:
 6617		break;
 6618	}
 6619
 6620	return NO_PRIV_STACK;
 6621}
 6622
 6623static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth)
 6624{
 6625	if (env->prog->jit_requested)
 6626		return round_up(stack_depth, 16);
 6627
 6628	/* round up to 32-bytes, since this is granularity
 6629	 * of interpreter stack size
 6630	 */
 6631	return round_up(max_t(u32, stack_depth, 1), 32);
 6632}
 6633
 6634/* starting from main bpf function walk all instructions of the function
 6635 * and recursively walk all callees that given function can call.
 6636 * Ignore jump and exit insns.
 6637 * Since recursion is prevented by check_cfg() this algorithm
 6638 * only needs a local stack of MAX_CALL_FRAMES to remember callsites
 6639 */
 6640static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 6641					 bool priv_stack_supported)
 6642{
 6643	struct bpf_subprog_info *subprog = env->subprog_info;
 6644	struct bpf_insn *insn = env->prog->insnsi;
 6645	int depth = 0, frame = 0, i, subprog_end, subprog_depth;
 6646	bool tail_call_reachable = false;
 6647	int ret_insn[MAX_CALL_FRAMES];
 6648	int ret_prog[MAX_CALL_FRAMES];
 6649	int j;
 6650
 6651	i = subprog[idx].start;
 6652	if (!priv_stack_supported)
 6653		subprog[idx].priv_stack_mode = NO_PRIV_STACK;
 6654process_func:
 6655	/* protect against potential stack overflow that might happen when
 6656	 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack
 6657	 * depth for such case down to 256 so that the worst case scenario
 6658	 * would result in 8k stack size (32 which is tailcall limit * 256 =
 6659	 * 8k).
 6660	 *
 6661	 * To get the idea what might happen, see an example:
 6662	 * func1 -> sub rsp, 128
 6663	 *  subfunc1 -> sub rsp, 256
 6664	 *  tailcall1 -> add rsp, 256
 6665	 *   func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320)
 6666	 *   subfunc2 -> sub rsp, 64
 6667	 *   subfunc22 -> sub rsp, 128
 6668	 *   tailcall2 -> add rsp, 128
 6669	 *    func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416)
 6670	 *
 6671	 * tailcall will unwind the current stack frame but it will not get rid
 6672	 * of caller's stack as shown on the example above.
 6673	 */
 6674	if (idx && subprog[idx].has_tail_call && depth >= 256) {
 6675		verbose(env,
 6676			"tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n",
 6677			depth);
 6678		return -EACCES;
 6679	}
 6680
 6681	subprog_depth = round_up_stack_depth(env, subprog[idx].stack_depth);
 6682	if (priv_stack_supported) {
 6683		/* Request private stack support only if the subprog stack
 6684		 * depth is no less than BPF_PRIV_STACK_MIN_SIZE. This is to
 6685		 * avoid jit penalty if the stack usage is small.
 6686		 */
 6687		if (subprog[idx].priv_stack_mode == PRIV_STACK_UNKNOWN &&
 6688		    subprog_depth >= BPF_PRIV_STACK_MIN_SIZE)
 6689			subprog[idx].priv_stack_mode = PRIV_STACK_ADAPTIVE;
 6690	}
 6691
 6692	if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) {
 6693		if (subprog_depth > MAX_BPF_STACK) {
 6694			verbose(env, "stack size of subprog %d is %d. Too large\n",
 6695				idx, subprog_depth);
 6696			return -EACCES;
 6697		}
 6698	} else {
 6699		depth += subprog_depth;
 6700		if (depth > MAX_BPF_STACK) {
 6701			verbose(env, "combined stack size of %d calls is %d. Too large\n",
 6702				frame + 1, depth);
 6703			return -EACCES;
 6704		}
 6705	}
 6706continue_func:
 6707	subprog_end = subprog[idx + 1].start;
 6708	for (; i < subprog_end; i++) {
 6709		int next_insn, sidx;
 6710
 6711		if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) {
 6712			bool err = false;
 6713
 6714			if (!is_bpf_throw_kfunc(insn + i))
 6715				continue;
 6716			if (subprog[idx].is_cb)
 6717				err = true;
 6718			for (int c = 0; c < frame && !err; c++) {
 6719				if (subprog[ret_prog[c]].is_cb) {
 6720					err = true;
 6721					break;
 6722				}
 6723			}
 6724			if (!err)
 6725				continue;
 6726			verbose(env,
 6727				"bpf_throw kfunc (insn %d) cannot be called from callback subprog %d\n",
 6728				i, idx);
 6729			return -EINVAL;
 6730		}
 6731
 6732		if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i))
 6733			continue;
 6734		/* remember insn and function to return to */
 6735		ret_insn[frame] = i + 1;
 6736		ret_prog[frame] = idx;
 6737
 6738		/* find the callee */
 6739		next_insn = i + insn[i].imm + 1;
 6740		sidx = find_subprog(env, next_insn);
 6741		if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn))
 6742			return -EFAULT;
 6743		if (subprog[sidx].is_async_cb) {
 6744			if (subprog[sidx].has_tail_call) {
 6745				verifier_bug(env, "subprog has tail_call and async cb");
 6746				return -EFAULT;
 6747			}
 6748			/* async callbacks don't increase bpf prog stack size unless called directly */
 6749			if (!bpf_pseudo_call(insn + i))
 6750				continue;
 6751			if (subprog[sidx].is_exception_cb) {
 6752				verbose(env, "insn %d cannot call exception cb directly", i);
 6753				return -EINVAL;
 6754			}
 6755		}
 6756		i = next_insn;
 6757		idx = sidx;
 6758		if (!priv_stack_supported)
 6759			subprog[idx].priv_stack_mode = NO_PRIV_STACK;
 6760
 6761		if (subprog[idx].has_tail_call)
 6762			tail_call_reachable = true;
 6763
 6764		frame++;
 6765		if (frame >= MAX_CALL_FRAMES) {
 6766			verbose(env, "the call stack of %d frames is too deep !\n",
 6767				frame);
 6768			return -E2BIG;
 6769		}
 6770		goto process_func;
 6771	}
 6772	/* if tail call got detected across bpf2bpf calls then mark each of the
 6773	 * currently present subprog frames as tail call reachable subprogs;
 6774	 * this info will be utilized by JIT so that we will be preserving the
 6775	 * tail call counter throughout bpf2bpf calls combined with tailcalls
 6776	 */
 6777	if (tail_call_reachable)
 6778		for (j = 0; j < frame; j++) {
 6779			if (subprog[ret_prog[j]].is_exception_cb) {
 6780				verbose(env, "cannot tail call within exception cb\n");
 6781				return -EINVAL;
 6782			}
 6783			subprog[ret_prog[j]].tail_call_reachable = true;
 6784		}
 6785	if (subprog[0].tail_call_reachable)
 6786		env->prog->aux->tail_call_reachable = true;
 6787
 6788	/* end of for() loop means the last insn of the 'subprog'
 6789	 * was reached. Doesn't matter whether it was JA or EXIT
 6790	 */
 6791	if (frame == 0)
 6792		return 0;
 6793	if (subprog[idx].priv_stack_mode != PRIV_STACK_ADAPTIVE)
 6794		depth -= round_up_stack_depth(env, subprog[idx].stack_depth);
 6795	frame--;
 6796	i = ret_insn[frame];
 6797	idx = ret_prog[frame];
 6798	goto continue_func;
 6799}
 6800
 6801static int check_max_stack_depth(struct bpf_verifier_env *env)
 6802{
 6803	enum priv_stack_mode priv_stack_mode = PRIV_STACK_UNKNOWN;
 6804	struct bpf_subprog_info *si = env->subprog_info;
 6805	bool priv_stack_supported;
 6806	int ret;
 6807
 6808	for (int i = 0; i < env->subprog_cnt; i++) {
 6809		if (si[i].has_tail_call) {
 6810			priv_stack_mode = NO_PRIV_STACK;
 6811			break;
 6812		}
 6813	}
 6814
 6815	if (priv_stack_mode == PRIV_STACK_UNKNOWN)
 6816		priv_stack_mode = bpf_enable_priv_stack(env->prog);
 6817
 6818	/* All async_cb subprogs use normal kernel stack. If a particular
 6819	 * subprog appears in both main prog and async_cb subtree, that
 6820	 * subprog will use normal kernel stack to avoid potential nesting.
 6821	 * The reverse subprog traversal ensures when main prog subtree is
 6822	 * checked, the subprogs appearing in async_cb subtrees are already
 6823	 * marked as using normal kernel stack, so stack size checking can
 6824	 * be done properly.
 6825	 */
 6826	for (int i = env->subprog_cnt - 1; i >= 0; i--) {
 6827		if (!i || si[i].is_async_cb) {
 6828			priv_stack_supported = !i && priv_stack_mode == PRIV_STACK_ADAPTIVE;
 6829			ret = check_max_stack_depth_subprog(env, i, priv_stack_supported);
 6830			if (ret < 0)
 6831				return ret;
 6832		}
 6833	}
 6834
 6835	for (int i = 0; i < env->subprog_cnt; i++) {
 6836		if (si[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) {
 6837			env->prog->aux->jits_use_priv_stack = true;
 6838			break;
 6839		}
 6840	}
 6841
 6842	return 0;
 6843}
 6844
 6845#ifndef CONFIG_BPF_JIT_ALWAYS_ON
 6846static int get_callee_stack_depth(struct bpf_verifier_env *env,
 6847				  const struct bpf_insn *insn, int idx)
 6848{
 6849	int start = idx + insn->imm + 1, subprog;
 6850
 6851	subprog = find_subprog(env, start);
 6852	if (verifier_bug_if(subprog < 0, env, "get stack depth: no program at insn %d", start))
 6853		return -EFAULT;
 6854	return env->subprog_info[subprog].stack_depth;
 6855}
 6856#endif
 6857
 6858static int __check_buffer_access(struct bpf_verifier_env *env,
 6859				 const char *buf_info,
 6860				 const struct bpf_reg_state *reg,
 6861				 int regno, int off, int size)
 6862{
 6863	if (off < 0) {
 6864		verbose(env,
 6865			"R%d invalid %s buffer access: off=%d, size=%d\n",
 6866			regno, buf_info, off, size);
 6867		return -EACCES;
 6868	}
 6869	if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
 6870		char tn_buf[48];
 6871
 6872		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
 6873		verbose(env,
 6874			"R%d invalid variable buffer offset: off=%d, var_off=%s\n",
 6875			regno, off, tn_buf);
 6876		return -EACCES;
 6877	}
 6878
 6879	return 0;
 6880}
 6881
 6882static int check_tp_buffer_access(struct bpf_verifier_env *env,
 6883				  const struct bpf_reg_state *reg,
 6884				  int regno, int off, int size)
 6885{
 6886	int err;
 6887
 6888	err = __check_buffer_access(env, "tracepoint", reg, regno, off, size);
 6889	if (err)
 6890		return err;
 6891
 6892	if (off + size > env->prog->aux->max_tp_access)
 6893		env->prog->aux->max_tp_access = off + size;
 6894
 6895	return 0;
 6896}
 6897
 6898static int check_buffer_access(struct bpf_verifier_env *env,
 6899			       const struct bpf_reg_state *reg,
 6900			       int regno, int off, int size,
 6901			       bool zero_size_allowed,
 6902			       u32 *max_access)
 6903{
 6904	const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr";
 6905	int err;
 6906
 6907	err = __check_buffer_access(env, buf_info, reg, regno, off, size);
 6908	if (err)
 6909		return err;
 6910
 6911	if (off + size > *max_access)
 6912		*max_access = off + size;
 6913
 6914	return 0;
 6915}
 6916
 6917/* BPF architecture zero extends alu32 ops into 64-bit registesr */
 6918static void zext_32_to_64(struct bpf_reg_state *reg)
 6919{
 6920	reg->var_off = tnum_subreg(reg->var_off);
 6921	__reg_assign_32_into_64(reg);
 6922}
 6923
 6924/* truncate register to smaller size (in bytes)
 6925 * must be called with size < BPF_REG_SIZE
 6926 */
 6927static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)
 6928{
 6929	u64 mask;
 6930
 6931	/* clear high bits in bit representation */
 6932	reg->var_off = tnum_cast(reg->var_off, size);
 6933
 6934	/* fix arithmetic bounds */
 6935	mask = ((u64)1 << (size * 8)) - 1;
 6936	if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) {
 6937		reg->umin_value &= mask;
 6938		reg->umax_value &= mask;
 6939	} else {
 6940		reg->umin_value = 0;
 6941		reg->umax_value = mask;
 6942	}
 6943	reg->smin_value = reg->umin_value;
 6944	reg->smax_value = reg->umax_value;
 6945
 6946	/* If size is smaller than 32bit register the 32bit register
 6947	 * values are also truncated so we push 64-bit bounds into
 6948	 * 32-bit bounds. Above were truncated < 32-bits already.
 6949	 */
 6950	if (size < 4)
 6951		__mark_reg32_unbounded(reg);
 6952
 6953	reg_bounds_sync(reg);
 6954}
 6955
 6956static void set_sext64_default_val(struct bpf_reg_state *reg, int size)
 6957{
 6958	if (size == 1) {
 6959		reg->smin_value = reg->s32_min_value = S8_MIN;
 6960		reg->smax_value = reg->s32_max_value = S8_MAX;
 6961	} else if (size == 2) {
 6962		reg->smin_value = reg->s32_min_value = S16_MIN;
 6963		reg->smax_value = reg->s32_max_value = S16_MAX;
 6964	} else {
 6965		/* size == 4 */
 6966		reg->smin_value = reg->s32_min_value = S32_MIN;
 6967		reg->smax_value = reg->s32_max_value = S32_MAX;
 6968	}
 6969	reg->umin_value = reg->u32_min_value = 0;
 6970	reg->umax_value = U64_MAX;
 6971	reg->u32_max_value = U32_MAX;
 6972	reg->var_off = tnum_unknown;
 6973}
 6974
 6975static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size)
 6976{
 6977	s64 init_s64_max, init_s64_min, s64_max, s64_min, u64_cval;
 6978	u64 top_smax_value, top_smin_value;
 6979	u64 num_bits = size * 8;
 6980
 6981	if (tnum_is_const(reg->var_off)) {
 6982		u64_cval = reg->var_off.value;
 6983		if (size == 1)
 6984			reg->var_off = tnum_const((s8)u64_cval);
 6985		else if (size == 2)
 6986			reg->var_off = tnum_const((s16)u64_cval);
 6987		else
 6988			/* size == 4 */
 6989			reg->var_off = tnum_const((s32)u64_cval);
 6990
 6991		u64_cval = reg->var_off.value;
 6992		reg->smax_value = reg->smin_value = u64_cval;
 6993		reg->umax_value = reg->umin_value = u64_cval;
 6994		reg->s32_max_value = reg->s32_min_value = u64_cval;
 6995		reg->u32_max_value = reg->u32_min_value = u64_cval;
 6996		return;
 6997	}
 6998
 6999	top_smax_value = ((u64)reg->smax_value >> num_bits) << num_bits;
 7000	top_smin_value = ((u64)reg->smin_value >> num_bits) << num_bits;
 7001
 7002	if (top_smax_value != top_smin_value)
 7003		goto out;
 7004
 7005	/* find the s64_min and s64_min after sign extension */
 7006	if (size == 1) {
 7007		init_s64_max = (s8)reg->smax_value;
 7008		init_s64_min = (s8)reg->smin_value;
 7009	} else if (size == 2) {
 7010		init_s64_max = (s16)reg->smax_value;
 7011		init_s64_min = (s16)reg->smin_value;
 7012	} else {
 7013		init_s64_max = (s32)reg->smax_value;
 7014		init_s64_min = (s32)reg->smin_value;
 7015	}
 7016
 7017	s64_max = max(init_s64_max, init_s64_min);
 7018	s64_min = min(init_s64_max, init_s64_min);
 7019
 7020	/* both of s64_max/s64_min positive or negative */
 7021	if ((s64_max >= 0) == (s64_min >= 0)) {
 7022		reg->s32_min_value = reg->smin_value = s64_min;
 7023		reg->s32_max_value = reg->smax_value = s64_max;
 7024		reg->u32_min_value = reg->umin_value = s64_min;
 7025		reg->u32_max_value = reg->umax_value = s64_max;
 7026		reg->var_off = tnum_range(s64_min, s64_max);
 7027		return;
 7028	}
 7029
 7030out:
 7031	set_sext64_default_val(reg, size);
 7032}
 7033
 7034static void set_sext32_default_val(struct bpf_reg_state *reg, int size)
 7035{
 7036	if (size == 1) {
 7037		reg->s32_min_value = S8_MIN;
 7038		reg->s32_max_value = S8_MAX;
 7039	} else {
 7040		/* size == 2 */
 7041		reg->s32_min_value = S16_MIN;
 7042		reg->s32_max_value = S16_MAX;
 7043	}
 7044	reg->u32_min_value = 0;
 7045	reg->u32_max_value = U32_MAX;
 7046	reg->var_off = tnum_subreg(tnum_unknown);
 7047}
 7048
 7049static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size)
 7050{
 7051	s32 init_s32_max, init_s32_min, s32_max, s32_min, u32_val;
 7052	u32 top_smax_value, top_smin_value;
 7053	u32 num_bits = size * 8;
 7054
 7055	if (tnum_is_const(reg->var_off)) {
 7056		u32_val = reg->var_off.value;
 7057		if (size == 1)
 7058			reg->var_off = tnum_const((s8)u32_val);
 7059		else
 7060			reg->var_off = tnum_const((s16)u32_val);
 7061
 7062		u32_val = reg->var_off.value;
 7063		reg->s32_min_value = reg->s32_max_value = u32_val;
 7064		reg->u32_min_value = reg->u32_max_value = u32_val;
 7065		return;
 7066	}
 7067
 7068	top_smax_value = ((u32)reg->s32_max_value >> num_bits) << num_bits;
 7069	top_smin_value = ((u32)reg->s32_min_value >> num_bits) << num_bits;
 7070
 7071	if (top_smax_value != top_smin_value)
 7072		goto out;
 7073
 7074	/* find the s32_min and s32_min after sign extension */
 7075	if (size == 1) {
 7076		init_s32_max = (s8)reg->s32_max_value;
 7077		init_s32_min = (s8)reg->s32_min_value;
 7078	} else {
 7079		/* size == 2 */
 7080		init_s32_max = (s16)reg->s32_max_value;
 7081		init_s32_min = (s16)reg->s32_min_value;
 7082	}
 7083	s32_max = max(init_s32_max, init_s32_min);
 7084	s32_min = min(init_s32_max, init_s32_min);
 7085
 7086	if ((s32_min >= 0) == (s32_max >= 0)) {
 7087		reg->s32_min_value = s32_min;
 7088		reg->s32_max_value = s32_max;
 7089		reg->u32_min_value = (u32)s32_min;
 7090		reg->u32_max_value = (u32)s32_max;
 7091		reg->var_off = tnum_subreg(tnum_range(s32_min, s32_max));
 7092		return;
 7093	}
 7094
 7095out:
 7096	set_sext32_default_val(reg, size);
 7097}
 7098
 7099static bool bpf_map_is_rdonly(const struct bpf_map *map)
 7100{
 7101	/* A map is considered read-only if the following condition are true:
 7102	 *
 7103	 * 1) BPF program side cannot change any of the map content. The
 7104	 *    BPF_F_RDONLY_PROG flag is throughout the lifetime of a map
 7105	 *    and was set at map creation time.
 7106	 * 2) The map value(s) have been initialized from user space by a
 7107	 *    loader and then "frozen", such that no new map update/delete
 7108	 *    operations from syscall side are possible for the rest of
 7109	 *    the map's lifetime from that point onwards.
 7110	 * 3) Any parallel/pending map update/delete operations from syscall
 7111	 *    side have been completed. Only after that point, it's safe to
 7112	 *    assume that map value(s) are immutable.
 7113	 */
 7114	return (map->map_flags & BPF_F_RDONLY_PROG) &&
 7115	       READ_ONCE(map->frozen) &&
 7116	       !bpf_map_write_active(map);
 7117}
 7118
 7119static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val,
 7120			       bool is_ldsx)
 7121{
 7122	void *ptr;
 7123	u64 addr;
 7124	int err;
 7125
 7126	err = map->ops->map_direct_value_addr(map, &addr, off);
 7127	if (err)
 7128		return err;
 7129	ptr = (void *)(long)addr + off;
 7130
 7131	switch (size) {
 7132	case sizeof(u8):
 7133		*val = is_ldsx ? (s64)*(s8 *)ptr : (u64)*(u8 *)ptr;
 7134		break;
 7135	case sizeof(u16):
 7136		*val = is_ldsx ? (s64)*(s16 *)ptr : (u64)*(u16 *)ptr;
 7137		break;
 7138	case sizeof(u32):
 7139		*val = is_ldsx ? (s64)*(s32 *)ptr : (u64)*(u32 *)ptr;
 7140		break;
 7141	case sizeof(u64):
 7142		*val = *(u64 *)ptr;
 7143		break;
 7144	default:
 7145		return -EINVAL;
 7146	}
 7147	return 0;
 7148}
 7149
 7150#define BTF_TYPE_SAFE_RCU(__type)  __PASTE(__type, __safe_rcu)
 7151#define BTF_TYPE_SAFE_RCU_OR_NULL(__type)  __PASTE(__type, __safe_rcu_or_null)
 7152#define BTF_TYPE_SAFE_TRUSTED(__type)  __PASTE(__type, __safe_trusted)
 7153#define BTF_TYPE_SAFE_TRUSTED_OR_NULL(__type)  __PASTE(__type, __safe_trusted_or_null)
 7154
 7155/*
 7156 * Allow list few fields as RCU trusted or full trusted.
 7157 * This logic doesn't allow mix tagging and will be removed once GCC supports
 7158 * btf_type_tag.
 7159 */
 7160
 7161/* RCU trusted: these fields are trusted in RCU CS and never NULL */
 7162BTF_TYPE_SAFE_RCU(struct task_struct) {
 7163	const cpumask_t *cpus_ptr;
 7164	struct css_set __rcu *cgroups;
 7165	struct task_struct __rcu *real_parent;
 7166	struct task_struct *group_leader;
 7167};
 7168
 7169BTF_TYPE_SAFE_RCU(struct cgroup) {
 7170	/* cgrp->kn is always accessible as documented in kernel/cgroup/cgroup.c */
 7171	struct kernfs_node *kn;
 7172};
 7173
 7174BTF_TYPE_SAFE_RCU(struct css_set) {
 7175	struct cgroup *dfl_cgrp;
 7176};
 7177
 7178BTF_TYPE_SAFE_RCU(struct cgroup_subsys_state) {
 7179	struct cgroup *cgroup;
 7180};
 7181
 7182/* RCU trusted: these fields are trusted in RCU CS and can be NULL */
 7183BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct) {
 7184	struct file __rcu *exe_file;
 7185#ifdef CONFIG_MEMCG
 7186	struct task_struct __rcu *owner;
 7187#endif
 7188};
 7189
 7190/* skb->sk, req->sk are not RCU protected, but we mark them as such
 7191 * because bpf prog accessible sockets are SOCK_RCU_FREE.
 7192 */
 7193BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff) {
 7194	struct sock *sk;
 7195};
 7196
 7197BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock) {
 7198	struct sock *sk;
 7199};
 7200
 7201/* full trusted: these fields are trusted even outside of RCU CS and never NULL */
 7202BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta) {
 7203	struct seq_file *seq;
 7204};
 7205
 7206BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task) {
 7207	struct bpf_iter_meta *meta;
 7208	struct task_struct *task;
 7209};
 7210
 7211BTF_TYPE_SAFE_TRUSTED(struct linux_binprm) {
 7212	struct file *file;
 7213};
 7214
 7215BTF_TYPE_SAFE_TRUSTED(struct file) {
 7216	struct inode *f_inode;
 7217};
 7218
 7219BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry) {
 7220	struct inode *d_inode;
 7221};
 7222
 7223BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket) {
 7224	struct sock *sk;
 7225};
 7226
 7227BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct vm_area_struct) {
 7228	struct mm_struct *vm_mm;
 7229	struct file *vm_file;
 7230};
 7231
 7232static bool type_is_rcu(struct bpf_verifier_env *env,
 7233			struct bpf_reg_state *reg,
 7234			const char *field_name, u32 btf_id)
 7235{
 7236	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct task_struct));
 7237	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct cgroup));
 7238	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct css_set));
 7239	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct cgroup_subsys_state));
 7240
 7241	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu");
 7242}
 7243
 7244static bool type_is_rcu_or_null(struct bpf_verifier_env *env,
 7245				struct bpf_reg_state *reg,
 7246				const char *field_name, u32 btf_id)
 7247{
 7248	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct));
 7249	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff));
 7250	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock));
 7251
 7252	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu_or_null");
 7253}
 7254
 7255static bool type_is_trusted(struct bpf_verifier_env *env,
 7256			    struct bpf_reg_state *reg,
 7257			    const char *field_name, u32 btf_id)
 7258{
 7259	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta));
 7260	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task));
 7261	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm));
 7262	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file));
 7263
 7264	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted");
 7265}
 7266
 7267static bool type_is_trusted_or_null(struct bpf_verifier_env *env,
 7268				    struct bpf_reg_state *reg,
 7269				    const char *field_name, u32 btf_id)
 7270{
 7271	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket));
 7272	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry));
 7273	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct vm_area_struct));
 7274
 7275	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id,
 7276					  "__safe_trusted_or_null");
 7277}
 7278
 7279static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
 7280				   struct bpf_reg_state *regs,
 7281				   int regno, int off, int size,
 7282				   enum bpf_access_type atype,
 7283				   int value_regno)
 7284{
 7285	struct bpf_reg_state *reg = regs + regno;
 7286	const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id);
 7287	const char *tname = btf_name_by_offset(reg->btf, t->name_off);
 7288	const char *field_name = NULL;
 7289	enum bpf_type_flag flag = 0;
 7290	u32 btf_id = 0;
 7291	int ret;
 7292
 7293	if (!env->allow_ptr_leaks) {
 7294		verbose(env,
 7295			"'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
 7296			tname);
 7297		return -EPERM;
 7298	}
 7299	if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) {
 7300		verbose(env,
 7301			"Cannot access kernel 'struct %s' from non-GPL compatible program\n",
 7302			tname);
 7303		return -EINVAL;
 7304	}
 7305	if (off < 0) {
 7306		verbose(env,
 7307			"R%d is ptr_%s invalid negative access: off=%d\n",
 7308			regno, tname, off);
 7309		return -EACCES;
 7310	}
 7311	if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
 7312		char tn_buf[48];
 7313
 7314		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
 7315		verbose(env,
 7316			"R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n",
 7317			regno, tname, off, tn_buf);
 7318		return -EACCES;
 7319	}
 7320
 7321	if (reg->type & MEM_USER) {
 7322		verbose(env,
 7323			"R%d is ptr_%s access user memory: off=%d\n",
 7324			regno, tname, off);
 7325		return -EACCES;
 7326	}
 7327
 7328	if (reg->type & MEM_PERCPU) {
 7329		verbose(env,
 7330			"R%d is ptr_%s access percpu memory: off=%d\n",
 7331			regno, tname, off);
 7332		return -EACCES;
 7333	}
 7334
 7335	if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) {
 7336		if (!btf_is_kernel(reg->btf)) {
 7337			verifier_bug(env, "reg->btf must be kernel btf");
 7338			return -EFAULT;
 7339		}
 7340		ret = env->ops->btf_struct_access(&env->log, reg, off, size);
 7341	} else {
 7342		/* Writes are permitted with default btf_struct_access for
 7343		 * program allocated objects (which always have ref_obj_id > 0),
 7344		 * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC.
 7345		 */
 7346		if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) {
 7347			verbose(env, "only read is supported\n");
 7348			return -EACCES;
 7349		}
 7350
 7351		if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) &&
 7352		    !(reg->type & MEM_RCU) && !reg->ref_obj_id) {
 7353			verifier_bug(env, "ref_obj_id for allocated object must be non-zero");
 7354			return -EFAULT;
 7355		}
 7356
 7357		ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag, &field_name);
 7358	}
 7359
 7360	if (ret < 0)
 7361		return ret;
 7362
 7363	if (ret != PTR_TO_BTF_ID) {
 7364		/* just mark; */
 7365
 7366	} else if (type_flag(reg->type) & PTR_UNTRUSTED) {
 7367		/* If this is an untrusted pointer, all pointers formed by walking it
 7368		 * also inherit the untrusted flag.
 7369		 */
 7370		flag = PTR_UNTRUSTED;
 7371
 7372	} else if (is_trusted_reg(reg) || is_rcu_reg(reg)) {
 7373		/* By default any pointer obtained from walking a trusted pointer is no
 7374		 * longer trusted, unless the field being accessed has explicitly been
 7375		 * marked as inheriting its parent's state of trust (either full or RCU).
 7376		 * For example:
 7377		 * 'cgroups' pointer is untrusted if task->cgroups dereference
 7378		 * happened in a sleepable program outside of bpf_rcu_read_lock()
 7379		 * section. In a non-sleepable program it's trusted while in RCU CS (aka MEM_RCU).
 7380		 * Note bpf_rcu_read_unlock() converts MEM_RCU pointers to PTR_UNTRUSTED.
 7381		 *
 7382		 * A regular RCU-protected pointer with __rcu tag can also be deemed
 7383		 * trusted if we are in an RCU CS. Such pointer can be NULL.
 7384		 */
 7385		if (type_is_trusted(env, reg, field_name, btf_id)) {
 7386			flag |= PTR_TRUSTED;
 7387		} else if (type_is_trusted_or_null(env, reg, field_name, btf_id)) {
 7388			flag |= PTR_TRUSTED | PTR_MAYBE_NULL;
 7389		} else if (in_rcu_cs(env) && !type_may_be_null(reg->type)) {
 7390			if (type_is_rcu(env, reg, field_name, btf_id)) {
 7391				/* ignore __rcu tag and mark it MEM_RCU */
 7392				flag |= MEM_RCU;
 7393			} else if (flag & MEM_RCU ||
 7394				   type_is_rcu_or_null(env, reg, field_name, btf_id)) {
 7395				/* __rcu tagged pointers can be NULL */
 7396				flag |= MEM_RCU | PTR_MAYBE_NULL;
 7397
 7398				/* We always trust them */
 7399				if (type_is_rcu_or_null(env, reg, field_name, btf_id) &&
 7400				    flag & PTR_UNTRUSTED)
 7401					flag &= ~PTR_UNTRUSTED;
 7402			} else if (flag & (MEM_PERCPU | MEM_USER)) {
 7403				/* keep as-is */
 7404			} else {
 7405				/* walking unknown pointers yields old deprecated PTR_TO_BTF_ID */
 7406				clear_trusted_flags(&flag);
 7407			}
 7408		} else {
 7409			/*
 7410			 * If not in RCU CS or MEM_RCU pointer can be NULL then
 7411			 * aggressively mark as untrusted otherwise such
 7412			 * pointers will be plain PTR_TO_BTF_ID without flags
 7413			 * and will be allowed to be passed into helpers for
 7414			 * compat reasons.
 7415			 */
 7416			flag = PTR_UNTRUSTED;
 7417		}
 7418	} else {
 7419		/* Old compat. Deprecated */
 7420		clear_trusted_flags(&flag);
 7421	}
 7422
 7423	if (atype == BPF_READ && value_regno >= 0) {
 7424		ret = mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag);
 7425		if (ret < 0)
 7426			return ret;
 7427	}
 7428
 7429	return 0;
 7430}
 7431
 7432static int check_ptr_to_map_access(struct bpf_verifier_env *env,
 7433				   struct bpf_reg_state *regs,
 7434				   int regno, int off, int size,
 7435				   enum bpf_access_type atype,
 7436				   int value_regno)
 7437{
 7438	struct bpf_reg_state *reg = regs + regno;
 7439	struct bpf_map *map = reg->map_ptr;
 7440	struct bpf_reg_state map_reg;
 7441	enum bpf_type_flag flag = 0;
 7442	const struct btf_type *t;
 7443	const char *tname;
 7444	u32 btf_id;
 7445	int ret;
 7446
 7447	if (!btf_vmlinux) {
 7448		verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n");
 7449		return -ENOTSUPP;
 7450	}
 7451
 7452	if (!map->ops->map_btf_id || !*map->ops->map_btf_id) {
 7453		verbose(env, "map_ptr access not supported for map type %d\n",
 7454			map->map_type);
 7455		return -ENOTSUPP;
 7456	}
 7457
 7458	t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id);
 7459	tname = btf_name_by_offset(btf_vmlinux, t->name_off);
 7460
 7461	if (!env->allow_ptr_leaks) {
 7462		verbose(env,
 7463			"'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
 7464			tname);
 7465		return -EPERM;
 7466	}
 7467
 7468	if (off < 0) {
 7469		verbose(env, "R%d is %s invalid negative access: off=%d\n",
 7470			regno, tname, off);
 7471		return -EACCES;
 7472	}
 7473
 7474	if (atype != BPF_READ) {
 7475		verbose(env, "only read from %s is supported\n", tname);
 7476		return -EACCES;
 7477	}
 7478
 7479	/* Simulate access to a PTR_TO_BTF_ID */
 7480	memset(&map_reg, 0, sizeof(map_reg));
 7481	ret = mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID,
 7482			      btf_vmlinux, *map->ops->map_btf_id, 0);
 7483	if (ret < 0)
 7484		return ret;
 7485	ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag, NULL);
 7486	if (ret < 0)
 7487		return ret;
 7488
 7489	if (value_regno >= 0) {
 7490		ret = mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag);
 7491		if (ret < 0)
 7492			return ret;
 7493	}
 7494
 7495	return 0;
 7496}
 7497
 7498/* Check that the stack access at the given offset is within bounds. The
 7499 * maximum valid offset is -1.
 7500 *
 7501 * The minimum valid offset is -MAX_BPF_STACK for writes, and
 7502 * -state->allocated_stack for reads.
 7503 */
 7504static int check_stack_slot_within_bounds(struct bpf_verifier_env *env,
 7505                                          s64 off,
 7506                                          struct bpf_func_state *state,
 7507                                          enum bpf_access_type t)
 7508{
 7509	int min_valid_off;
 7510
 7511	if (t == BPF_WRITE || env->allow_uninit_stack)
 7512		min_valid_off = -MAX_BPF_STACK;
 7513	else
 7514		min_valid_off = -state->allocated_stack;
 7515
 7516	if (off < min_valid_off || off > -1)
 7517		return -EACCES;
 7518	return 0;
 7519}
 7520
 7521/* Check that the stack access at 'regno + off' falls within the maximum stack
 7522 * bounds.
 7523 *
 7524 * 'off' includes `regno->offset`, but not its dynamic part (if any).
 7525 */
 7526static int check_stack_access_within_bounds(
 7527		struct bpf_verifier_env *env,
 7528		int regno, int off, int access_size,
 7529		enum bpf_access_type type)
 7530{
 7531	struct bpf_reg_state *regs = cur_regs(env);
 7532	struct bpf_reg_state *reg = regs + regno;
 7533	struct bpf_func_state *state = func(env, reg);
 7534	s64 min_off, max_off;
 7535	int err;
 7536	char *err_extra;
 7537
 7538	if (type == BPF_READ)
 7539		err_extra = " read from";
 7540	else
 7541		err_extra = " write to";
 7542
 7543	if (tnum_is_const(reg->var_off)) {
 7544		min_off = (s64)reg->var_off.value + off;
 7545		max_off = min_off + access_size;
 7546	} else {
 7547		if (reg->smax_value >= BPF_MAX_VAR_OFF ||
 7548		    reg->smin_value <= -BPF_MAX_VAR_OFF) {
 7549			verbose(env, "invalid unbounded variable-offset%s stack R%d\n",
 7550				err_extra, regno);
 7551			return -EACCES;
 7552		}
 7553		min_off = reg->smin_value + off;
 7554		max_off = reg->smax_value + off + access_size;
 7555	}
 7556
 7557	err = check_stack_slot_within_bounds(env, min_off, state, type);
 7558	if (!err && max_off > 0)
 7559		err = -EINVAL; /* out of stack access into non-negative offsets */
 7560	if (!err && access_size < 0)
 7561		/* access_size should not be negative (or overflow an int); others checks
 7562		 * along the way should have prevented such an access.
 7563		 */
 7564		err = -EFAULT; /* invalid negative access size; integer overflow? */
 7565
 7566	if (err) {
 7567		if (tnum_is_const(reg->var_off)) {
 7568			verbose(env, "invalid%s stack R%d off=%d size=%d\n",
 7569				err_extra, regno, off, access_size);
 7570		} else {
 7571			char tn_buf[48];
 7572
 7573			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
 7574			verbose(env, "invalid variable-offset%s stack R%d var_off=%s off=%d size=%d\n",
 7575				err_extra, regno, tn_buf, off, access_size);
 7576		}
 7577		return err;
 7578	}
 7579
 7580	/* Note that there is no stack access with offset zero, so the needed stack
 7581	 * size is -min_off, not -min_off+1.
 7582	 */
 7583	return grow_stack_state(env, state, -min_off /* size */);
 7584}
 7585
 7586static bool get_func_retval_range(struct bpf_prog *prog,
 7587				  struct bpf_retval_range *range)
 7588{
 7589	if (prog->type == BPF_PROG_TYPE_LSM &&
 7590		prog->expected_attach_type == BPF_LSM_MAC &&
 7591		!bpf_lsm_get_retval_range(prog, range)) {
 7592		return true;
 7593	}
 7594	return false;
 7595}
 7596
 7597/* check whether memory at (regno + off) is accessible for t = (read | write)
 7598 * if t==write, value_regno is a register which value is stored into memory
 7599 * if t==read, value_regno is a register which will receive the value from memory
 7600 * if t==write && value_regno==-1, some unknown value is stored into memory
 7601 * if t==read && value_regno==-1, don't care what we read from memory
 7602 */
 7603static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,
 7604			    int off, int bpf_size, enum bpf_access_type t,
 7605			    int value_regno, bool strict_alignment_once, bool is_ldsx)
 7606{
 7607	struct bpf_reg_state *regs = cur_regs(env);
 7608	struct bpf_reg_state *reg = regs + regno;
 7609	int size, err = 0;
 7610
 7611	size = bpf_size_to_bytes(bpf_size);
 7612	if (size < 0)
 7613		return size;
 7614
 7615	/* alignment checks will add in reg->off themselves */
 7616	err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);
 7617	if (err)
 7618		return err;
 7619
 7620	/* for access checks, reg->off is just part of off */
 7621	off += reg->off;
 7622
 7623	if (reg->type == PTR_TO_MAP_KEY) {
 7624		if (t == BPF_WRITE) {
 7625			verbose(env, "write to change key R%d not allowed\n", regno);
 7626			return -EACCES;
 7627		}
 7628
 7629		err = check_mem_region_access(env, regno, off, size,
 7630					      reg->map_ptr->key_size, false);
 7631		if (err)
 7632			return err;
 7633		if (value_regno >= 0)
 7634			mark_reg_unknown(env, regs, value_regno);
 7635	} else if (reg->type == PTR_TO_MAP_VALUE) {
 7636		struct btf_field *kptr_field = NULL;
 7637
 7638		if (t == BPF_WRITE && value_regno >= 0 &&
 7639		    is_pointer_value(env, value_regno)) {
 7640			verbose(env, "R%d leaks addr into map\n", value_regno);
 7641			return -EACCES;
 7642		}
 7643		err = check_map_access_type(env, regno, off, size, t);
 7644		if (err)
 7645			return err;
 7646		err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT);
 7647		if (err)
 7648			return err;
 7649		if (tnum_is_const(reg->var_off))
 7650			kptr_field = btf_record_find(reg->map_ptr->record,
 7651						     off + reg->var_off.value, BPF_KPTR | BPF_UPTR);
 7652		if (kptr_field) {
 7653			err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field);
 7654		} else if (t == BPF_READ && value_regno >= 0) {
 7655			struct bpf_map *map = reg->map_ptr;
 7656
 7657			/*
 7658			 * If map is read-only, track its contents as scalars,
 7659			 * unless it is an insn array (see the special case below)
 7660			 */
 7661			if (tnum_is_const(reg->var_off) &&
 7662			    bpf_map_is_rdonly(map) &&
 7663			    map->ops->map_direct_value_addr &&
 7664			    map->map_type != BPF_MAP_TYPE_INSN_ARRAY) {
 7665				int map_off = off + reg->var_off.value;
 7666				u64 val = 0;
 7667
 7668				err = bpf_map_direct_read(map, map_off, size,
 7669							  &val, is_ldsx);
 7670				if (err)
 7671					return err;
 7672
 7673				regs[value_regno].type = SCALAR_VALUE;
 7674				__mark_reg_known(&regs[value_regno], val);
 7675			} else if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) {
 7676				if (bpf_size != BPF_DW) {
 7677					verbose(env, "Invalid read of %d bytes from insn_array\n",
 7678						     size);
 7679					return -EACCES;
 7680				}
 7681				copy_register_state(&regs[value_regno], reg);
 7682				regs[value_regno].type = PTR_TO_INSN;
 7683			} else {
 7684				mark_reg_unknown(env, regs, value_regno);
 7685			}
 7686		}
 7687	} else if (base_type(reg->type) == PTR_TO_MEM) {
 7688		bool rdonly_mem = type_is_rdonly_mem(reg->type);
 7689		bool rdonly_untrusted = rdonly_mem && (reg->type & PTR_UNTRUSTED);
 7690
 7691		if (type_may_be_null(reg->type)) {
 7692			verbose(env, "R%d invalid mem access '%s'\n", regno,
 7693				reg_type_str(env, reg->type));
 7694			return -EACCES;
 7695		}
 7696
 7697		if (t == BPF_WRITE && rdonly_mem) {
 7698			verbose(env, "R%d cannot write into %s\n",
 7699				regno, reg_type_str(env, reg->type));
 7700			return -EACCES;
 7701		}
 7702
 7703		if (t == BPF_WRITE && value_regno >= 0 &&
 7704		    is_pointer_value(env, value_regno)) {
 7705			verbose(env, "R%d leaks addr into mem\n", value_regno);
 7706			return -EACCES;
 7707		}
 7708
 7709		/*
 7710		 * Accesses to untrusted PTR_TO_MEM are done through probe
 7711		 * instructions, hence no need to check bounds in that case.
 7712		 */
 7713		if (!rdonly_untrusted)
 7714			err = check_mem_region_access(env, regno, off, size,
 7715						      reg->mem_size, false);
 7716		if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem))
 7717			mark_reg_unknown(env, regs, value_regno);
 7718	} else if (reg->type == PTR_TO_CTX) {
 7719		struct bpf_retval_range range;
 7720		struct bpf_insn_access_aux info = {
 7721			.reg_type = SCALAR_VALUE,
 7722			.is_ldsx = is_ldsx,
 7723			.log = &env->log,
 7724		};
 7725
 7726		if (t == BPF_WRITE && value_regno >= 0 &&
 7727		    is_pointer_value(env, value_regno)) {
 7728			verbose(env, "R%d leaks addr into ctx\n", value_regno);
 7729			return -EACCES;
 7730		}
 7731
 7732		err = check_ptr_off_reg(env, reg, regno);
 7733		if (err < 0)
 7734			return err;
 7735
 7736		err = check_ctx_access(env, insn_idx, off, size, t, &info);
 7737		if (err)
 7738			verbose_linfo(env, insn_idx, "; ");
 7739		if (!err && t == BPF_READ && value_regno >= 0) {
 7740			/* ctx access returns either a scalar, or a
 7741			 * PTR_TO_PACKET[_META,_END]. In the latter
 7742			 * case, we know the offset is zero.
 7743			 */
 7744			if (info.reg_type == SCALAR_VALUE) {
 7745				if (info.is_retval && get_func_retval_range(env->prog, &range)) {
 7746					err = __mark_reg_s32_range(env, regs, value_regno,
 7747								   range.minval, range.maxval);
 7748					if (err)
 7749						return err;
 7750				} else {
 7751					mark_reg_unknown(env, regs, value_regno);
 7752				}
 7753			} else {
 7754				mark_reg_known_zero(env, regs,
 7755						    value_regno);
 7756				if (type_may_be_null(info.reg_type))
 7757					regs[value_regno].id = ++env->id_gen;
 7758				/* A load of ctx field could have different
 7759				 * actual load size with the one encoded in the
 7760				 * insn. When the dst is PTR, it is for sure not
 7761				 * a sub-register.
 7762				 */
 7763				regs[value_regno].subreg_def = DEF_NOT_SUBREG;
 7764				if (base_type(info.reg_type) == PTR_TO_BTF_ID) {
 7765					regs[value_regno].btf = info.btf;
 7766					regs[value_regno].btf_id = info.btf_id;
 7767					regs[value_regno].ref_obj_id = info.ref_obj_id;
 7768				}
 7769			}
 7770			regs[value_regno].type = info.reg_type;
 7771		}
 7772
 7773	} else if (reg->type == PTR_TO_STACK) {
 7774		/* Basic bounds checks. */
 7775		err = check_stack_access_within_bounds(env, regno, off, size, t);
 7776		if (err)
 7777			return err;
 7778
 7779		if (t == BPF_READ)
 7780			err = check_stack_read(env, regno, off, size,
 7781					       value_regno);
 7782		else
 7783			err = check_stack_write(env, regno, off, size,
 7784						value_regno, insn_idx);
 7785	} else if (reg_is_pkt_pointer(reg)) {
 7786		if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
 7787			verbose(env, "cannot write into packet\n");
 7788			return -EACCES;
 7789		}
 7790		if (t == BPF_WRITE && value_regno >= 0 &&
 7791		    is_pointer_value(env, value_regno)) {
 7792			verbose(env, "R%d leaks addr into packet\n",
 7793				value_regno);
 7794			return -EACCES;
 7795		}
 7796		err = check_packet_access(env, regno, off, size, false);
 7797		if (!err && t == BPF_READ && value_regno >= 0)
 7798			mark_reg_unknown(env, regs, value_regno);
 7799	} else if (reg->type == PTR_TO_FLOW_KEYS) {
 7800		if (t == BPF_WRITE && value_regno >= 0 &&
 7801		    is_pointer_value(env, value_regno)) {
 7802			verbose(env, "R%d leaks addr into flow keys\n",
 7803				value_regno);
 7804			return -EACCES;
 7805		}
 7806
 7807		err = check_flow_keys_access(env, off, size);
 7808		if (!err && t == BPF_READ && value_regno >= 0)
 7809			mark_reg_unknown(env, regs, value_regno);
 7810	} else if (type_is_sk_pointer(reg->type)) {
 7811		if (t == BPF_WRITE) {
 7812			verbose(env, "R%d cannot write into %s\n",
 7813				regno, reg_type_str(env, reg->type));
 7814			return -EACCES;
 7815		}
 7816		err = check_sock_access(env, insn_idx, regno, off, size, t);
 7817		if (!err && value_regno >= 0)
 7818			mark_reg_unknown(env, regs, value_regno);
 7819	} else if (reg->type == PTR_TO_TP_BUFFER) {
 7820		err = check_tp_buffer_access(env, reg, regno, off, size);
 7821		if (!err && t == BPF_READ && value_regno >= 0)
 7822			mark_reg_unknown(env, regs, value_regno);
 7823	} else if (base_type(reg->type) == PTR_TO_BTF_ID &&
 7824		   !type_may_be_null(reg->type)) {
 7825		err = check_ptr_to_btf_access(env, regs, regno, off, size, t,
 7826					      value_regno);
 7827	} else if (reg->type == CONST_PTR_TO_MAP) {
 7828		err = check_ptr_to_map_access(env, regs, regno, off, size, t,
 7829					      value_regno);
 7830	} else if (base_type(reg->type) == PTR_TO_BUF &&
 7831		   !type_may_be_null(reg->type)) {
 7832		bool rdonly_mem = type_is_rdonly_mem(reg->type);
 7833		u32 *max_access;
 7834
 7835		if (rdonly_mem) {
 7836			if (t == BPF_WRITE) {
 7837				verbose(env, "R%d cannot write into %s\n",
 7838					regno, reg_type_str(env, reg->type));
 7839				return -EACCES;
 7840			}
 7841			max_access = &env->prog->aux->max_rdonly_access;
 7842		} else {
 7843			max_access = &env->prog->aux->max_rdwr_access;
 7844		}
 7845
 7846		err = check_buffer_access(env, reg, regno, off, size, false,
 7847					  max_access);
 7848
 7849		if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ))
 7850			mark_reg_unknown(env, regs, value_regno);
 7851	} else if (reg->type == PTR_TO_ARENA) {
 7852		if (t == BPF_READ && value_regno >= 0)
 7853			mark_reg_unknown(env, regs, value_regno);
 7854	} else {
 7855		verbose(env, "R%d invalid mem access '%s'\n", regno,
 7856			reg_type_str(env, reg->type));
 7857		return -EACCES;
 7858	}
 7859
 7860	if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ &&
 7861	    regs[value_regno].type == SCALAR_VALUE) {
 7862		if (!is_ldsx)
 7863			/* b/h/w load zero-extends, mark upper bits as known 0 */
 7864			coerce_reg_to_size(&regs[value_regno], size);
 7865		else
 7866			coerce_reg_to_size_sx(&regs[value_regno], size);
 7867	}
 7868	return err;
 7869}
 7870
 7871static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,
 7872			     bool allow_trust_mismatch);
 7873
 7874static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn,
 7875			  bool strict_alignment_once, bool is_ldsx,
 7876			  bool allow_trust_mismatch, const char *ctx)
 7877{
 7878	struct bpf_reg_state *regs = cur_regs(env);
 7879	enum bpf_reg_type src_reg_type;
 7880	int err;
 7881
 7882	/* check src operand */
 7883	err = check_reg_arg(env, insn->src_reg, SRC_OP);
 7884	if (err)
 7885		return err;
 7886
 7887	/* check dst operand */
 7888	err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
 7889	if (err)
 7890		return err;
 7891
 7892	src_reg_type = regs[insn->src_reg].type;
 7893
 7894	/* Check if (src_reg + off) is readable. The state of dst_reg will be
 7895	 * updated by this call.
 7896	 */
 7897	err = check_mem_access(env, env->insn_idx, insn->src_reg, insn->off,
 7898			       BPF_SIZE(insn->code), BPF_READ, insn->dst_reg,
 7899			       strict_alignment_once, is_ldsx);
 7900	err = err ?: save_aux_ptr_type(env, src_reg_type,
 7901				       allow_trust_mismatch);
 7902	err = err ?: reg_bounds_sanity_check(env, &regs[insn->dst_reg], ctx);
 7903
 7904	return err;
 7905}
 7906
 7907static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn,
 7908			   bool strict_alignment_once)
 7909{
 7910	struct bpf_reg_state *regs = cur_regs(env);
 7911	enum bpf_reg_type dst_reg_type;
 7912	int err;
 7913
 7914	/* check src1 operand */
 7915	err = check_reg_arg(env, insn->src_reg, SRC_OP);
 7916	if (err)
 7917		return err;
 7918
 7919	/* check src2 operand */
 7920	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
 7921	if (err)
 7922		return err;
 7923
 7924	dst_reg_type = regs[insn->dst_reg].type;
 7925
 7926	/* Check if (dst_reg + off) is writeable. */
 7927	err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,
 7928			       BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg,
 7929			       strict_alignment_once, false);
 7930	err = err ?: save_aux_ptr_type(env, dst_reg_type, false);
 7931
 7932	return err;
 7933}
 7934
 7935static int check_atomic_rmw(struct bpf_verifier_env *env,
 7936			    struct bpf_insn *insn)
 7937{
 7938	int load_reg;
 7939	int err;
 7940
 7941	if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) {
 7942		verbose(env, "invalid atomic operand size\n");
 7943		return -EINVAL;
 7944	}
 7945
 7946	/* check src1 operand */
 7947	err = check_reg_arg(env, insn->src_reg, SRC_OP);
 7948	if (err)
 7949		return err;
 7950
 7951	/* check src2 operand */
 7952	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
 7953	if (err)
 7954		return err;
 7955
 7956	if (insn->imm == BPF_CMPXCHG) {
 7957		/* Check comparison of R0 with memory location */
 7958		const u32 aux_reg = BPF_REG_0;
 7959
 7960		err = check_reg_arg(env, aux_reg, SRC_OP);
 7961		if (err)
 7962			return err;
 7963
 7964		if (is_pointer_value(env, aux_reg)) {
 7965			verbose(env, "R%d leaks addr into mem\n", aux_reg);
 7966			return -EACCES;
 7967		}
 7968	}
 7969
 7970	if (is_pointer_value(env, insn->src_reg)) {
 7971		verbose(env, "R%d leaks addr into mem\n", insn->src_reg);
 7972		return -EACCES;
 7973	}
 7974
 7975	if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) {
 7976		verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
 7977			insn->dst_reg,
 7978			reg_type_str(env, reg_state(env, insn->dst_reg)->type));
 7979		return -EACCES;
 7980	}
 7981
 7982	if (insn->imm & BPF_FETCH) {
 7983		if (insn->imm == BPF_CMPXCHG)
 7984			load_reg = BPF_REG_0;
 7985		else
 7986			load_reg = insn->src_reg;
 7987
 7988		/* check and record load of old value */
 7989		err = check_reg_arg(env, load_reg, DST_OP);
 7990		if (err)
 7991			return err;
 7992	} else {
 7993		/* This instruction accesses a memory location but doesn't
 7994		 * actually load it into a register.
 7995		 */
 7996		load_reg = -1;
 7997	}
 7998
 7999	/* Check whether we can read the memory, with second call for fetch
 8000	 * case to simulate the register fill.
 8001	 */
 8002	err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,
 8003			       BPF_SIZE(insn->code), BPF_READ, -1, true, false);
 8004	if (!err && load_reg >= 0)
 8005		err = check_mem_access(env, env->insn_idx, insn->dst_reg,
 8006				       insn->off, BPF_SIZE(insn->code),
 8007				       BPF_READ, load_reg, true, false);
 8008	if (err)
 8009		return err;
 8010
 8011	if (is_arena_reg(env, insn->dst_reg)) {
 8012		err = save_aux_ptr_type(env, PTR_TO_ARENA, false);
 8013		if (err)
 8014			return err;
 8015	}
 8016	/* Check whether we can write into the same memory. */
 8017	err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,
 8018			       BPF_SIZE(insn->code), BPF_WRITE, -1, true, false);
 8019	if (err)
 8020		return err;
 8021	return 0;
 8022}
 8023
 8024static int check_atomic_load(struct bpf_verifier_env *env,
 8025			     struct bpf_insn *insn)
 8026{
 8027	int err;
 8028
 8029	err = check_load_mem(env, insn, true, false, false, "atomic_load");
 8030	if (err)
 8031		return err;
 8032
 8033	if (!atomic_ptr_type_ok(env, insn->src_reg, insn)) {
 8034		verbose(env, "BPF_ATOMIC loads from R%d %s is not allowed\n",
 8035			insn->src_reg,
 8036			reg_type_str(env, reg_state(env, insn->src_reg)->type));
 8037		return -EACCES;
 8038	}
 8039
 8040	return 0;
 8041}
 8042
 8043static int check_atomic_store(struct bpf_verifier_env *env,
 8044			      struct bpf_insn *insn)
 8045{
 8046	int err;
 8047
 8048	err = check_store_reg(env, insn, true);
 8049	if (err)
 8050		return err;
 8051
 8052	if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) {
 8053		verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
 8054			insn->dst_reg,
 8055			reg_type_str(env, reg_state(env, insn->dst_reg)->type));
 8056		return -EACCES;
 8057	}
 8058
 8059	return 0;
 8060}
 8061
 8062static int check_atomic(struct bpf_verifier_env *env, struct bpf_insn *insn)
 8063{
 8064	switch (insn->imm) {
 8065	case BPF_ADD:
 8066	case BPF_ADD | BPF_FETCH:
 8067	case BPF_AND:
 8068	case BPF_AND | BPF_FETCH:
 8069	case BPF_OR:
 8070	case BPF_OR | BPF_FETCH:
 8071	case BPF_XOR:
 8072	case BPF_XOR | BPF_FETCH:
 8073	case BPF_XCHG:
 8074	case BPF_CMPXCHG:
 8075		return check_atomic_rmw(env, insn);
 8076	case BPF_LOAD_ACQ:
 8077		if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) {
 8078			verbose(env,
 8079				"64-bit load-acquires are only supported on 64-bit arches\n");
 8080			return -EOPNOTSUPP;
 8081		}
 8082		return check_atomic_load(env, insn);
 8083	case BPF_STORE_REL:
 8084		if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) {
 8085			verbose(env,
 8086				"64-bit store-releases are only supported on 64-bit arches\n");
 8087			return -EOPNOTSUPP;
 8088		}
 8089		return check_atomic_store(env, insn);
 8090	default:
 8091		verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n",
 8092			insn->imm);
 8093		return -EINVAL;
 8094	}
 8095}
 8096
 8097/* When register 'regno' is used to read the stack (either directly or through
 8098 * a helper function) make sure that it's within stack boundary and, depending
 8099 * on the access type and privileges, that all elements of the stack are
 8100 * initialized.
 8101 *
 8102 * 'off' includes 'regno->off', but not its dynamic part (if any).
 8103 *
 8104 * All registers that have been spilled on the stack in the slots within the
 8105 * read offsets are marked as read.
 8106 */
 8107static int check_stack_range_initialized(
 8108		struct bpf_verifier_env *env, int regno, int off,
 8109		int access_size, bool zero_size_allowed,
 8110		enum bpf_access_type type, struct bpf_call_arg_meta *meta)
 8111{
 8112	struct bpf_reg_state *reg = reg_state(env, regno);
 8113	struct bpf_func_state *state = func(env, reg);
 8114	int err, min_off, max_off, i, j, slot, spi;
 8115	/* Some accesses can write anything into the stack, others are
 8116	 * read-only.
 8117	 */
 8118	bool clobber = false;
 8119
 8120	if (access_size == 0 && !zero_size_allowed) {
 8121		verbose(env, "invalid zero-sized read\n");
 8122		return -EACCES;
 8123	}
 8124
 8125	if (type == BPF_WRITE)
 8126		clobber = true;
 8127
 8128	err = check_stack_access_within_bounds(env, regno, off, access_size, type);
 8129	if (err)
 8130		return err;
 8131
 8132
 8133	if (tnum_is_const(reg->var_off)) {
 8134		min_off = max_off = reg->var_off.value + off;
 8135	} else {
 8136		/* Variable offset is prohibited for unprivileged mode for
 8137		 * simplicity since it requires corresponding support in
 8138		 * Spectre masking for stack ALU.
 8139		 * See also retrieve_ptr_limit().
 8140		 */
 8141		if (!env->bypass_spec_v1) {
 8142			char tn_buf[48];
 8143
 8144			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
 8145			verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n",
 8146				regno, tn_buf);
 8147			return -EACCES;
 8148		}
 8149		/* Only initialized buffer on stack is allowed to be accessed
 8150		 * with variable offset. With uninitialized buffer it's hard to
 8151		 * guarantee that whole memory is marked as initialized on
 8152		 * helper return since specific bounds are unknown what may
 8153		 * cause uninitialized stack leaking.
 8154		 */
 8155		if (meta && meta->raw_mode)
 8156			meta = NULL;
 8157
 8158		min_off = reg->smin_value + off;
 8159		max_off = reg->smax_value + off;
 8160	}
 8161
 8162	if (meta && meta->raw_mode) {
 8163		/* Ensure we won't be overwriting dynptrs when simulating byte
 8164		 * by byte access in check_helper_call using meta.access_size.
 8165		 * This would be a problem if we have a helper in the future
 8166		 * which takes:
 8167		 *
 8168		 *	helper(uninit_mem, len, dynptr)
 8169		 *
 8170		 * Now, uninint_mem may overlap with dynptr pointer. Hence, it
 8171		 * may end up writing to dynptr itself when touching memory from
 8172		 * arg 1. This can be relaxed on a case by case basis for known
 8173		 * safe cases, but reject due to the possibilitiy of aliasing by
 8174		 * default.
 8175		 */
 8176		for (i = min_off; i < max_off + access_size; i++) {
 8177			int stack_off = -i - 1;
 8178
 8179			spi = __get_spi(i);
 8180			/* raw_mode may write past allocated_stack */
 8181			if (state->allocated_stack <= stack_off)
 8182				continue;
 8183			if (state->stack[spi].slot_type[stack_off % BPF_REG_SIZE] == STACK_DYNPTR) {
 8184				verbose(env, "potential write to dynptr at off=%d disallowed\n", i);
 8185				return -EACCES;
 8186			}
 8187		}
 8188		meta->access_size = access_size;
 8189		meta->regno = regno;
 8190		return 0;
 8191	}
 8192
 8193	for (i = min_off; i < max_off + access_size; i++) {
 8194		u8 *stype;
 8195
 8196		slot = -i - 1;
 8197		spi = slot / BPF_REG_SIZE;
 8198		if (state->allocated_stack <= slot) {
 8199			verbose(env, "allocated_stack too small\n");
 8200			return -EFAULT;
 8201		}
 8202
 8203		stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
 8204		if (*stype == STACK_MISC)
 8205			goto mark;
 8206		if ((*stype == STACK_ZERO) ||
 8207		    (*stype == STACK_INVALID && env->allow_uninit_stack)) {
 8208			if (clobber) {
 8209				/* helper can write anything into the stack */
 8210				*stype = STACK_MISC;
 8211			}
 8212			goto mark;
 8213		}
 8214
 8215		if (is_spilled_reg(&state->stack[spi]) &&
 8216		    (state->stack[spi].spilled_ptr.type == SCALAR_VALUE ||
 8217		     env->allow_ptr_leaks)) {
 8218			if (clobber) {
 8219				__mark_reg_unknown(env, &state->stack[spi].spilled_ptr);
 8220				for (j = 0; j < BPF_REG_SIZE; j++)
 8221					scrub_spilled_slot(&state->stack[spi].slot_type[j]);
 8222			}
 8223			goto mark;
 8224		}
 8225
 8226		if (tnum_is_const(reg->var_off)) {
 8227			verbose(env, "invalid read from stack R%d off %d+%d size %d\n",
 8228				regno, min_off, i - min_off, access_size);
 8229		} else {
 8230			char tn_buf[48];
 8231
 8232			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
 8233			verbose(env, "invalid read from stack R%d var_off %s+%d size %d\n",
 8234				regno, tn_buf, i - min_off, access_size);
 8235		}
 8236		return -EACCES;
 8237mark:
 8238		/* reading any byte out of 8-byte 'spill_slot' will cause
 8239		 * the whole slot to be marked as 'read'
 8240		 */
 8241		err = bpf_mark_stack_read(env, reg->frameno, env->insn_idx, BIT(spi));
 8242		if (err)
 8243			return err;
 8244		/* We do not call bpf_mark_stack_write(), as we can not
 8245		 * be sure that whether stack slot is written to or not. Hence,
 8246		 * we must still conservatively propagate reads upwards even if
 8247		 * helper may write to the entire memory range.
 8248		 */
 8249	}
 8250	return 0;
 8251}
 8252
 8253static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
 8254				   int access_size, enum bpf_access_type access_type,
 8255				   bool zero_size_allowed,
 8256				   struct bpf_call_arg_meta *meta)
 8257{
 8258	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
 8259	u32 *max_access;
 8260
 8261	switch (base_type(reg->type)) {
 8262	case PTR_TO_PACKET:
 8263	case PTR_TO_PACKET_META:
 8264		return check_packet_access(env, regno, reg->off, access_size,
 8265					   zero_size_allowed);
 8266	case PTR_TO_MAP_KEY:
 8267		if (access_type == BPF_WRITE) {
 8268			verbose(env, "R%d cannot write into %s\n", regno,
 8269				reg_type_str(env, reg->type));
 8270			return -EACCES;
 8271		}
 8272		return check_mem_region_access(env, regno, reg->off, access_size,
 8273					       reg->map_ptr->key_size, false);
 8274	case PTR_TO_MAP_VALUE:
 8275		if (check_map_access_type(env, regno, reg->off, access_size, access_type))
 8276			return -EACCES;
 8277		return check_map_access(env, regno, reg->off, access_size,
 8278					zero_size_allowed, ACCESS_HELPER);
 8279	case PTR_TO_MEM:
 8280		if (type_is_rdonly_mem(reg->type)) {
 8281			if (access_type == BPF_WRITE) {
 8282				verbose(env, "R%d cannot write into %s\n", regno,
 8283					reg_type_str(env, reg->type));
 8284				return -EACCES;
 8285			}
 8286		}
 8287		return check_mem_region_access(env, regno, reg->off,
 8288					       access_size, reg->mem_size,
 8289					       zero_size_allowed);
 8290	case PTR_TO_BUF:
 8291		if (type_is_rdonly_mem(reg->type)) {
 8292			if (access_type == BPF_WRITE) {
 8293				verbose(env, "R%d cannot write into %s\n", regno,
 8294					reg_type_str(env, reg->type));
 8295				return -EACCES;
 8296			}
 8297
 8298			max_access = &env->prog->aux->max_rdonly_access;
 8299		} else {
 8300			max_access = &env->prog->aux->max_rdwr_access;
 8301		}
 8302		return check_buffer_access(env, reg, regno, reg->off,
 8303					   access_size, zero_size_allowed,
 8304					   max_access);
 8305	case PTR_TO_STACK:
 8306		return check_stack_range_initialized(
 8307				env,
 8308				regno, reg->off, access_size,
 8309				zero_size_allowed, access_type, meta);
 8310	case PTR_TO_BTF_ID:
 8311		return check_ptr_to_btf_access(env, regs, regno, reg->off,
 8312					       access_size, BPF_READ, -1);
 8313	case PTR_TO_CTX:
 8314		/* in case the function doesn't know how to access the context,
 8315		 * (because we are in a program of type SYSCALL for example), we
 8316		 * can not statically check its size.
 8317		 * Dynamically check it now.
 8318		 */
 8319		if (!env->ops->convert_ctx_access) {
 8320			int offset = access_size - 1;
 8321
 8322			/* Allow zero-byte read from PTR_TO_CTX */
 8323			if (access_size == 0)
 8324				return zero_size_allowed ? 0 : -EACCES;
 8325
 8326			return check_mem_access(env, env->insn_idx, regno, offset, BPF_B,
 8327						access_type, -1, false, false);
 8328		}
 8329
 8330		fallthrough;
 8331	default: /* scalar_value or invalid ptr */
 8332		/* Allow zero-byte read from NULL, regardless of pointer type */
 8333		if (zero_size_allowed && access_size == 0 &&
 8334		    register_is_null(reg))
 8335			return 0;
 8336
 8337		verbose(env, "R%d type=%s ", regno,
 8338			reg_type_str(env, reg->type));
 8339		verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK));
 8340		return -EACCES;
 8341	}
 8342}
 8343
 8344/* verify arguments to helpers or kfuncs consisting of a pointer and an access
 8345 * size.
 8346 *
 8347 * @regno is the register containing the access size. regno-1 is the register
 8348 * containing the pointer.
 8349 */
 8350static int check_mem_size_reg(struct bpf_verifier_env *env,
 8351			      struct bpf_reg_state *reg, u32 regno,
 8352			      enum bpf_access_type access_type,
 8353			      bool zero_size_allowed,
 8354			      struct bpf_call_arg_meta *meta)
 8355{
 8356	int err;
 8357
 8358	/* This is used to refine r0 return value bounds for helpers
 8359	 * that enforce this value as an upper bound on return values.
 8360	 * See do_refine_retval_range() for helpers that can refine
 8361	 * the return value. C type of helper is u32 so we pull register
 8362	 * bound from umax_value however, if negative verifier errors
 8363	 * out. Only upper bounds can be learned because retval is an
 8364	 * int type and negative retvals are allowed.
 8365	 */
 8366	meta->msize_max_value = reg->umax_value;
 8367
 8368	/* The register is SCALAR_VALUE; the access check happens using
 8369	 * its boundaries. For unprivileged variable accesses, disable
 8370	 * raw mode so that the program is required to initialize all
 8371	 * the memory that the helper could just partially fill up.
 8372	 */
 8373	if (!tnum_is_const(reg->var_off))
 8374		meta = NULL;
 8375
 8376	if (reg->smin_value < 0) {
 8377		verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n",
 8378			regno);
 8379		return -EACCES;
 8380	}
 8381
 8382	if (reg->umin_value == 0 && !zero_size_allowed) {
 8383		verbose(env, "R%d invalid zero-sized read: u64=[%lld,%lld]\n",
 8384			regno, reg->umin_value, reg->umax_value);
 8385		return -EACCES;
 8386	}
 8387
 8388	if (reg->umax_value >= BPF_MAX_VAR_SIZ) {
 8389		verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n",
 8390			regno);
 8391		return -EACCES;
 8392	}
 8393	err = check_helper_mem_access(env, regno - 1, reg->umax_value,
 8394				      access_type, zero_size_allowed, meta);
 8395	if (!err)
 8396		err = mark_chain_precision(env, regno);
 8397	return err;
 8398}
 8399
 8400static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
 8401			 u32 regno, u32 mem_size)
 8402{
 8403	bool may_be_null = type_may_be_null(reg->type);
 8404	struct bpf_reg_state saved_reg;
 8405	int err;
 8406
 8407	if (register_is_null(reg))
 8408		return 0;
 8409
 8410	/* Assuming that the register contains a value check if the memory
 8411	 * access is safe. Temporarily save and restore the register's state as
 8412	 * the conversion shouldn't be visible to a caller.
 8413	 */
 8414	if (may_be_null) {
 8415		saved_reg = *reg;
 8416		mark_ptr_not_null_reg(reg);
 8417	}
 8418
 8419	err = check_helper_mem_access(env, regno, mem_size, BPF_READ, true, NULL);
 8420	err = err ?: check_helper_mem_access(env, regno, mem_size, BPF_WRITE, true, NULL);
 8421
 8422	if (may_be_null)
 8423		*reg = saved_reg;
 8424
 8425	return err;
 8426}
 8427
 8428static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
 8429				    u32 regno)
 8430{
 8431	struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1];
 8432	bool may_be_null = type_may_be_null(mem_reg->type);
 8433	struct bpf_reg_state saved_reg;
 8434	struct bpf_call_arg_meta meta;
 8435	int err;
 8436
 8437	WARN_ON_ONCE(regno < BPF_REG_2 || regno > BPF_REG_5);
 8438
 8439	memset(&meta, 0, sizeof(meta));
 8440
 8441	if (may_be_null) {
 8442		saved_reg = *mem_reg;
 8443		mark_ptr_not_null_reg(mem_reg);
 8444	}
 8445
 8446	err = check_mem_size_reg(env, reg, regno, BPF_READ, true, &meta);
 8447	err = err ?: check_mem_size_reg(env, reg, regno, BPF_WRITE, true, &meta);
 8448
 8449	if (may_be_null)
 8450		*mem_reg = saved_reg;
 8451
 8452	return err;
 8453}
 8454
 8455enum {
 8456	PROCESS_SPIN_LOCK = (1 << 0),
 8457	PROCESS_RES_LOCK  = (1 << 1),
 8458	PROCESS_LOCK_IRQ  = (1 << 2),
 8459};
 8460
 8461/* Implementation details:
 8462 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL.
 8463 * bpf_obj_new returns PTR_TO_BTF_ID | MEM_ALLOC | PTR_MAYBE_NULL.
 8464 * Two bpf_map_lookups (even with the same key) will have different reg->id.
 8465 * Two separate bpf_obj_new will also have different reg->id.
 8466 * For traditional PTR_TO_MAP_VALUE or PTR_TO_BTF_ID | MEM_ALLOC, the verifier
 8467 * clears reg->id after value_or_null->value transition, since the verifier only
 8468 * cares about the range of access to valid map value pointer and doesn't care
 8469 * about actual address of the map element.
 8470 * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps
 8471 * reg->id > 0 after value_or_null->value transition. By doing so
 8472 * two bpf_map_lookups will be considered two different pointers that
 8473 * point to different bpf_spin_locks. Likewise for pointers to allocated objects
 8474 * returned from bpf_obj_new.
 8475 * The verifier allows taking only one bpf_spin_lock at a time to avoid
 8476 * dead-locks.
 8477 * Since only one bpf_spin_lock is allowed the checks are simpler than
 8478 * reg_is_refcounted() logic. The verifier needs to remember only
 8479 * one spin_lock instead of array of acquired_refs.
 8480 * env->cur_state->active_locks remembers which map value element or allocated
 8481 * object got locked and clears it after bpf_spin_unlock.
 8482 */
 8483static int process_spin_lock(struct bpf_verifier_env *env, int regno, int flags)
 8484{
 8485	bool is_lock = flags & PROCESS_SPIN_LOCK, is_res_lock = flags & PROCESS_RES_LOCK;
 8486	const char *lock_str = is_res_lock ? "bpf_res_spin" : "bpf_spin";
 8487	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
 8488	struct bpf_verifier_state *cur = env->cur_state;
 8489	bool is_const = tnum_is_const(reg->var_off);
 8490	bool is_irq = flags & PROCESS_LOCK_IRQ;
 8491	u64 val = reg->var_off.value;
 8492	struct bpf_map *map = NULL;
 8493	struct btf *btf = NULL;
 8494	struct btf_record *rec;
 8495	u32 spin_lock_off;
 8496	int err;
 8497
 8498	if (!is_const) {
 8499		verbose(env,
 8500			"R%d doesn't have constant offset. %s_lock has to be at the constant offset\n",
 8501			regno, lock_str);
 8502		return -EINVAL;
 8503	}
 8504	if (reg->type == PTR_TO_MAP_VALUE) {
 8505		map = reg->map_ptr;
 8506		if (!map->btf) {
 8507			verbose(env,
 8508				"map '%s' has to have BTF in order to use %s_lock\n",
 8509				map->name, lock_str);
 8510			return -EINVAL;
 8511		}
 8512	} else {
 8513		btf = reg->btf;
 8514	}
 8515
 8516	rec = reg_btf_record(reg);
 8517	if (!btf_record_has_field(rec, is_res_lock ? BPF_RES_SPIN_LOCK : BPF_SPIN_LOCK)) {
 8518		verbose(env, "%s '%s' has no valid %s_lock\n", map ? "map" : "local",
 8519			map ? map->name : "kptr", lock_str);
 8520		return -EINVAL;
 8521	}
 8522	spin_lock_off = is_res_lock ? rec->res_spin_lock_off : rec->spin_lock_off;
 8523	if (spin_lock_off != val + reg->off) {
 8524		verbose(env, "off %lld doesn't point to 'struct %s_lock' that is at %d\n",
 8525			val + reg->off, lock_str, spin_lock_off);
 8526		return -EINVAL;
 8527	}
 8528	if (is_lock) {
 8529		void *ptr;
 8530		int type;
 8531
 8532		if (map)
 8533			ptr = map;
 8534		else
 8535			ptr = btf;
 8536
 8537		if (!is_res_lock && cur->active_locks) {
 8538			if (find_lock_state(env->cur_state, REF_TYPE_LOCK, 0, NULL)) {
 8539				verbose(env,
 8540					"Locking two bpf_spin_locks are not allowed\n");
 8541				return -EINVAL;
 8542			}
 8543		} else if (is_res_lock && cur->active_locks) {
 8544			if (find_lock_state(env->cur_state, REF_TYPE_RES_LOCK | REF_TYPE_RES_LOCK_IRQ, reg->id, ptr)) {
 8545				verbose(env, "Acquiring the same lock again, AA deadlock detected\n");
 8546				return -EINVAL;
 8547			}
 8548		}
 8549
 8550		if (is_res_lock && is_irq)
 8551			type = REF_TYPE_RES_LOCK_IRQ;
 8552		else if (is_res_lock)
 8553			type = REF_TYPE_RES_LOCK;
 8554		else
 8555			type = REF_TYPE_LOCK;
 8556		err = acquire_lock_state(env, env->insn_idx, type, reg->id, ptr);
 8557		if (err < 0) {
 8558			verbose(env, "Failed to acquire lock state\n");
 8559			return err;
 8560		}
 8561	} else {
 8562		void *ptr;
 8563		int type;
 8564
 8565		if (map)
 8566			ptr = map;
 8567		else
 8568			ptr = btf;
 8569
 8570		if (!cur->active_locks) {
 8571			verbose(env, "%s_unlock without taking a lock\n", lock_str);
 8572			return -EINVAL;
 8573		}
 8574
 8575		if (is_res_lock && is_irq)
 8576			type = REF_TYPE_RES_LOCK_IRQ;
 8577		else if (is_res_lock)
 8578			type = REF_TYPE_RES_LOCK;
 8579		else
 8580			type = REF_TYPE_LOCK;
 8581		if (!find_lock_state(cur, type, reg->id, ptr)) {
 8582			verbose(env, "%s_unlock of different lock\n", lock_str);
 8583			return -EINVAL;
 8584		}
 8585		if (reg->id != cur->active_lock_id || ptr != cur->active_lock_ptr) {
 8586			verbose(env, "%s_unlock cannot be out of order\n", lock_str);
 8587			return -EINVAL;
 8588		}
 8589		if (release_lock_state(cur, type, reg->id, ptr)) {
 8590			verbose(env, "%s_unlock of different lock\n", lock_str);
 8591			return -EINVAL;
 8592		}
 8593
 8594		invalidate_non_owning_refs(env);
 8595	}
 8596	return 0;
 8597}
 8598
 8599/* Check if @regno is a pointer to a specific field in a map value */
 8600static int check_map_field_pointer(struct bpf_verifier_env *env, u32 regno,
 8601				   enum btf_field_type field_type)
 8602{
 8603	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
 8604	bool is_const = tnum_is_const(reg->var_off);
 8605	struct bpf_map *map = reg->map_ptr;
 8606	u64 val = reg->var_off.value;
 8607	const char *struct_name = btf_field_type_name(field_type);
 8608	int field_off = -1;
 8609
 8610	if (!is_const) {
 8611		verbose(env,
 8612			"R%d doesn't have constant offset. %s has to be at the constant offset\n",
 8613			regno, struct_name);
 8614		return -EINVAL;
 8615	}
 8616	if (!map->btf) {
 8617		verbose(env, "map '%s' has to have BTF in order to use %s\n", map->name,
 8618			struct_name);
 8619		return -EINVAL;
 8620	}
 8621	if (!btf_record_has_field(map->record, field_type)) {
 8622		verbose(env, "map '%s' has no valid %s\n", map->name, struct_name);
 8623		return -EINVAL;
 8624	}
 8625	switch (field_type) {
 8626	case BPF_TIMER:
 8627		field_off = map->record->timer_off;
 8628		break;
 8629	case BPF_TASK_WORK:
 8630		field_off = map->record->task_work_off;
 8631		break;
 8632	case BPF_WORKQUEUE:
 8633		field_off = map->record->wq_off;
 8634		break;
 8635	default:
 8636		verifier_bug(env, "unsupported BTF field type: %s\n", struct_name);
 8637		return -EINVAL;
 8638	}
 8639	if (field_off != val + reg->off) {
 8640		verbose(env, "off %lld doesn't point to 'struct %s' that is at %d\n",
 8641			val + reg->off, struct_name, field_off);
 8642		return -EINVAL;
 8643	}
 8644	return 0;
 8645}
 8646
 8647static int process_timer_func(struct bpf_verifier_env *env, int regno,
 8648			      struct bpf_call_arg_meta *meta)
 8649{
 8650	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
 8651	struct bpf_map *map = reg->map_ptr;
 8652	int err;
 8653
 8654	err = check_map_field_pointer(env, regno, BPF_TIMER);
 8655	if (err)
 8656		return err;
 8657
 8658	if (meta->map_ptr) {
 8659		verifier_bug(env, "Two map pointers in a timer helper");
 8660		return -EFAULT;
 8661	}
 8662	if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
 8663		verbose(env, "bpf_timer cannot be used for PREEMPT_RT.\n");
 8664		return -EOPNOTSUPP;
 8665	}
 8666	meta->map_uid = reg->map_uid;
 8667	meta->map_ptr = map;
 8668	return 0;
 8669}
 8670
 8671static int process_wq_func(struct bpf_verifier_env *env, int regno,
 8672			   struct bpf_kfunc_call_arg_meta *meta)
 8673{
 8674	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
 8675	struct bpf_map *map = reg->map_ptr;
 8676	int err;
 8677
 8678	err = check_map_field_pointer(env, regno, BPF_WORKQUEUE);
 8679	if (err)
 8680		return err;
 8681
 8682	if (meta->map.ptr) {
 8683		verifier_bug(env, "Two map pointers in a bpf_wq helper");
 8684		return -EFAULT;
 8685	}
 8686
 8687	meta->map.uid = reg->map_uid;
 8688	meta->map.ptr = map;
 8689	return 0;
 8690}
 8691
 8692static int process_task_work_func(struct bpf_verifier_env *env, int regno,
 8693				  struct bpf_kfunc_call_arg_meta *meta)
 8694{
 8695	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
 8696	struct bpf_map *map = reg->map_ptr;
 8697	int err;
 8698
 8699	err = check_map_field_pointer(env, regno, BPF_TASK_WORK);
 8700	if (err)
 8701		return err;
 8702
 8703	if (meta->map.ptr) {
 8704		verifier_bug(env, "Two map pointers in a bpf_task_work helper");
 8705		return -EFAULT;
 8706	}
 8707	meta->map.uid = reg->map_uid;
 8708	meta->map.ptr = map;
 8709	return 0;
 8710}
 8711
 8712static int process_kptr_func(struct bpf_verifier_env *env, int regno,
 8713			     struct bpf_call_arg_meta *meta)
 8714{
 8715	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
 8716	struct btf_field *kptr_field;
 8717	struct bpf_map *map_ptr;
 8718	struct btf_record *rec;
 8719	u32 kptr_off;
 8720
 8721	if (type_is_ptr_alloc_obj(reg->type)) {
 8722		rec = reg_btf_record(reg);
 8723	} else { /* PTR_TO_MAP_VALUE */
 8724		map_ptr = reg->map_ptr;
 8725		if (!map_ptr->btf) {
 8726			verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n",
 8727				map_ptr->name);
 8728			return -EINVAL;
 8729		}
 8730		rec = map_ptr->record;
 8731		meta->map_ptr = map_ptr;
 8732	}
 8733
 8734	if (!tnum_is_const(reg->var_off)) {
 8735		verbose(env,
 8736			"R%d doesn't have constant offset. kptr has to be at the constant offset\n",
 8737			regno);
 8738		return -EINVAL;
 8739	}
 8740
 8741	if (!btf_record_has_field(rec, BPF_KPTR)) {
 8742		verbose(env, "R%d has no valid kptr\n", regno);
 8743		return -EINVAL;
 8744	}
 8745
 8746	kptr_off = reg->off + reg->var_off.value;
 8747	kptr_field = btf_record_find(rec, kptr_off, BPF_KPTR);
 8748	if (!kptr_field) {
 8749		verbose(env, "off=%d doesn't point to kptr\n", kptr_off);
 8750		return -EACCES;
 8751	}
 8752	if (kptr_field->type != BPF_KPTR_REF && kptr_field->type != BPF_KPTR_PERCPU) {
 8753		verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off);
 8754		return -EACCES;
 8755	}
 8756	meta->kptr_field = kptr_field;
 8757	return 0;
 8758}
 8759
 8760/* There are two register types representing a bpf_dynptr, one is PTR_TO_STACK
 8761 * which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR.
 8762 *
 8763 * In both cases we deal with the first 8 bytes, but need to mark the next 8
 8764 * bytes as STACK_DYNPTR in case of PTR_TO_STACK. In case of
 8765 * CONST_PTR_TO_DYNPTR, we are guaranteed to get the beginning of the object.
 8766 *
 8767 * Mutability of bpf_dynptr is at two levels, one is at the level of struct
 8768 * bpf_dynptr itself, i.e. whether the helper is receiving a pointer to struct
 8769 * bpf_dynptr or pointer to const struct bpf_dynptr. In the former case, it can
 8770 * mutate the view of the dynptr and also possibly destroy it. In the latter
 8771 * case, it cannot mutate the bpf_dynptr itself but it can still mutate the
 8772 * memory that dynptr points to.
 8773 *
 8774 * The verifier will keep track both levels of mutation (bpf_dynptr's in
 8775 * reg->type and the memory's in reg->dynptr.type), but there is no support for
 8776 * readonly dynptr view yet, hence only the first case is tracked and checked.
 8777 *
 8778 * This is consistent with how C applies the const modifier to a struct object,
 8779 * where the pointer itself inside bpf_dynptr becomes const but not what it
 8780 * points to.
 8781 *
 8782 * Helpers which do not mutate the bpf_dynptr set MEM_RDONLY in their argument
 8783 * type, and declare it as 'const struct bpf_dynptr *' in their prototype.
 8784 */
 8785static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx,
 8786			       enum bpf_arg_type arg_type, int clone_ref_obj_id)
 8787{
 8788	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
 8789	int err;
 8790
 8791	if (reg->type != PTR_TO_STACK && reg->type != CONST_PTR_TO_DYNPTR) {
 8792		verbose(env,
 8793			"arg#%d expected pointer to stack or const struct bpf_dynptr\n",
 8794			regno - 1);
 8795		return -EINVAL;
 8796	}
 8797
 8798	/* MEM_UNINIT and MEM_RDONLY are exclusive, when applied to an
 8799	 * ARG_PTR_TO_DYNPTR (or ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_*):
 8800	 */
 8801	if ((arg_type & (MEM_UNINIT | MEM_RDONLY)) == (MEM_UNINIT | MEM_RDONLY)) {
 8802		verifier_bug(env, "misconfigured dynptr helper type flags");
 8803		return -EFAULT;
 8804	}
 8805
 8806	/*  MEM_UNINIT - Points to memory that is an appropriate candidate for
 8807	 *		 constructing a mutable bpf_dynptr object.
 8808	 *
 8809	 *		 Currently, this is only possible with PTR_TO_STACK
 8810	 *		 pointing to a region of at least 16 bytes which doesn't
 8811	 *		 contain an existing bpf_dynptr.
 8812	 *
 8813	 *  MEM_RDONLY - Points to a initialized bpf_dynptr that will not be
 8814	 *		 mutated or destroyed. However, the memory it points to
 8815	 *		 may be mutated.
 8816	 *
 8817	 *  None       - Points to a initialized dynptr that can be mutated and
 8818	 *		 destroyed, including mutation of the memory it points
 8819	 *		 to.
 8820	 */
 8821	if (arg_type & MEM_UNINIT) {
 8822		int i;
 8823
 8824		if (!is_dynptr_reg_valid_uninit(env, reg)) {
 8825			verbose(env, "Dynptr has to be an uninitialized dynptr\n");
 8826			return -EINVAL;
 8827		}
 8828
 8829		/* we write BPF_DW bits (8 bytes) at a time */
 8830		for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) {
 8831			err = check_mem_access(env, insn_idx, regno,
 8832					       i, BPF_DW, BPF_WRITE, -1, false, false);
 8833			if (err)
 8834				return err;
 8835		}
 8836
 8837		err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, clone_ref_obj_id);
 8838	} else /* MEM_RDONLY and None case from above */ {
 8839		/* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */
 8840		if (reg->type == CONST_PTR_TO_DYNPTR && !(arg_type & MEM_RDONLY)) {
 8841			verbose(env, "cannot pass pointer to const bpf_dynptr, the helper mutates it\n");
 8842			return -EINVAL;
 8843		}
 8844
 8845		if (!is_dynptr_reg_valid_init(env, reg)) {
 8846			verbose(env,
 8847				"Expected an initialized dynptr as arg #%d\n",
 8848				regno - 1);
 8849			return -EINVAL;
 8850		}
 8851
 8852		/* Fold modifiers (in this case, MEM_RDONLY) when checking expected type */
 8853		if (!is_dynptr_type_expected(env, reg, arg_type & ~MEM_RDONLY)) {
 8854			verbose(env,
 8855				"Expected a dynptr of type %s as arg #%d\n",
 8856				dynptr_type_str(arg_to_dynptr_type(arg_type)), regno - 1);
 8857			return -EINVAL;
 8858		}
 8859
 8860		err = mark_dynptr_read(env, reg);
 8861	}
 8862	return err;
 8863}
 8864
 8865static u32 iter_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int spi)
 8866{
 8867	struct bpf_func_state *state = func(env, reg);
 8868
 8869	return state->stack[spi].spilled_ptr.ref_obj_id;
 8870}
 8871
 8872static bool is_iter_kfunc(struct bpf_kfunc_call_arg_meta *meta)
 8873{
 8874	return meta->kfunc_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY);
 8875}
 8876
 8877static bool is_iter_new_kfunc(struct bpf_kfunc_call_arg_meta *meta)
 8878{
 8879	return meta->kfunc_flags & KF_ITER_NEW;
 8880}
 8881
 8882static bool is_iter_next_kfunc(struct bpf_kfunc_call_arg_meta *meta)
 8883{
 8884	return meta->kfunc_flags & KF_ITER_NEXT;
 8885}
 8886
 8887static bool is_iter_destroy_kfunc(struct bpf_kfunc_call_arg_meta *meta)
 8888{
 8889	return meta->kfunc_flags & KF_ITER_DESTROY;
 8890}
 8891
 8892static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg_idx,
 8893			      const struct btf_param *arg)
 8894{
 8895	/* btf_check_iter_kfuncs() guarantees that first argument of any iter
 8896	 * kfunc is iter state pointer
 8897	 */
 8898	if (is_iter_kfunc(meta))
 8899		return arg_idx == 0;
 8900
 8901	/* iter passed as an argument to a generic kfunc */
 8902	return btf_param_match_suffix(meta->btf, arg, "__iter");
 8903}
 8904
 8905static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_idx,
 8906			    struct bpf_kfunc_call_arg_meta *meta)
 8907{
 8908	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
 8909	const struct btf_type *t;
 8910	int spi, err, i, nr_slots, btf_id;
 8911
 8912	if (reg->type != PTR_TO_STACK) {
 8913		verbose(env, "arg#%d expected pointer to an iterator on stack\n", regno - 1);
 8914		return -EINVAL;
 8915	}
 8916
 8917	/* For iter_{new,next,destroy} functions, btf_check_iter_kfuncs()
 8918	 * ensures struct convention, so we wouldn't need to do any BTF
 8919	 * validation here. But given iter state can be passed as a parameter
 8920	 * to any kfunc, if arg has "__iter" suffix, we need to be a bit more
 8921	 * conservative here.
 8922	 */
 8923	btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, regno - 1);
 8924	if (btf_id < 0) {
 8925		verbose(env, "expected valid iter pointer as arg #%d\n", regno - 1);
 8926		return -EINVAL;
 8927	}
 8928	t = btf_type_by_id(meta->btf, btf_id);
 8929	nr_slots = t->size / BPF_REG_SIZE;
 8930
 8931	if (is_iter_new_kfunc(meta)) {
 8932		/* bpf_iter_<type>_new() expects pointer to uninit iter state */
 8933		if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) {
 8934			verbose(env, "expected uninitialized iter_%s as arg #%d\n",
 8935				iter_type_str(meta->btf, btf_id), regno - 1);
 8936			return -EINVAL;
 8937		}
 8938
 8939		for (i = 0; i < nr_slots * 8; i += BPF_REG_SIZE) {
 8940			err = check_mem_access(env, insn_idx, regno,
 8941					       i, BPF_DW, BPF_WRITE, -1, false, false);
 8942			if (err)
 8943				return err;
 8944		}
 8945
 8946		err = mark_stack_slots_iter(env, meta, reg, insn_idx, meta->btf, btf_id, nr_slots);
 8947		if (err)
 8948			return err;
 8949	} else {
 8950		/* iter_next() or iter_destroy(), as well as any kfunc
 8951		 * accepting iter argument, expect initialized iter state
 8952		 */
 8953		err = is_iter_reg_valid_init(env, reg, meta->btf, btf_id, nr_slots);
 8954		switch (err) {
 8955		case 0:
 8956			break;
 8957		case -EINVAL:
 8958			verbose(env, "expected an initialized iter_%s as arg #%d\n",
 8959				iter_type_str(meta->btf, btf_id), regno - 1);
 8960			return err;
 8961		case -EPROTO:
 8962			verbose(env, "expected an RCU CS when using %s\n", meta->func_name);
 8963			return err;
 8964		default:
 8965			return err;
 8966		}
 8967
 8968		spi = iter_get_spi(env, reg, nr_slots);
 8969		if (spi < 0)
 8970			return spi;
 8971
 8972		err = mark_iter_read(env, reg, spi, nr_slots);
 8973		if (err)
 8974			return err;
 8975
 8976		/* remember meta->iter info for process_iter_next_call() */
 8977		meta->iter.spi = spi;
 8978		meta->iter.frameno = reg->frameno;
 8979		meta->ref_obj_id = iter_ref_obj_id(env, reg, spi);
 8980
 8981		if (is_iter_destroy_kfunc(meta)) {
 8982			err = unmark_stack_slots_iter(env, reg, nr_slots);
 8983			if (err)
 8984				return err;
 8985		}
 8986	}
 8987
 8988	return 0;
 8989}
 8990
 8991/* Look for a previous loop entry at insn_idx: nearest parent state
 8992 * stopped at insn_idx with callsites matching those in cur->frame.
 8993 */
 8994static struct bpf_verifier_state *find_prev_entry(struct bpf_verifier_env *env,
 8995						  struct bpf_verifier_state *cur,
 8996						  int insn_idx)
 8997{
 8998	struct bpf_verifier_state_list *sl;
 8999	struct bpf_verifier_state *st;
 9000	struct list_head *pos, *head;
 9001
 9002	/* Explored states are pushed in stack order, most recent states come first */
 9003	head = explored_state(env, insn_idx);
 9004	list_for_each(pos, head) {
 9005		sl = container_of(pos, struct bpf_verifier_state_list, node);
 9006		/* If st->branches != 0 state is a part of current DFS verification path,
 9007		 * hence cur & st for a loop.
 9008		 */
 9009		st = &sl->state;
 9010		if (st->insn_idx == insn_idx && st->branches && same_callsites(st, cur) &&
 9011		    st->dfs_depth < cur->dfs_depth)
 9012			return st;
 9013	}
 9014
 9015	return NULL;
 9016}
 9017
 9018static void reset_idmap_scratch(struct bpf_verifier_env *env);
 9019static bool regs_exact(const struct bpf_reg_state *rold,
 9020		       const struct bpf_reg_state *rcur,
 9021		       struct bpf_idmap *idmap);
 9022
 9023static void maybe_widen_reg(struct bpf_verifier_env *env,
 9024			    struct bpf_reg_state *rold, struct bpf_reg_state *rcur,
 9025			    struct bpf_idmap *idmap)
 9026{
 9027	if (rold->type != SCALAR_VALUE)
 9028		return;
 9029	if (rold->type != rcur->type)
 9030		return;
 9031	if (rold->precise || rcur->precise || regs_exact(rold, rcur, idmap))
 9032		return;
 9033	__mark_reg_unknown(env, rcur);
 9034}
 9035
 9036static int widen_imprecise_scalars(struct bpf_verifier_env *env,
 9037				   struct bpf_verifier_state *old,
 9038				   struct bpf_verifier_state *cur)
 9039{
 9040	struct bpf_func_state *fold, *fcur;
 9041	int i, fr, num_slots;
 9042
 9043	reset_idmap_scratch(env);
 9044	for (fr = old->curframe; fr >= 0; fr--) {
 9045		fold = old->frame[fr];
 9046		fcur = cur->frame[fr];
 9047
 9048		for (i = 0; i < MAX_BPF_REG; i++)
 9049			maybe_widen_reg(env,
 9050					&fold->regs[i],
 9051					&fcur->regs[i],
 9052					&env->idmap_scratch);
 9053
 9054		num_slots = min(fold->allocated_stack / BPF_REG_SIZE,
 9055				fcur->allocated_stack / BPF_REG_SIZE);
 9056		for (i = 0; i < num_slots; i++) {
 9057			if (!is_spilled_reg(&fold->stack[i]) ||
 9058			    !is_spilled_reg(&fcur->stack[i]))
 9059				continue;
 9060
 9061			maybe_widen_reg(env,
 9062					&fold->stack[i].spilled_ptr,
 9063					&fcur->stack[i].spilled_ptr,
 9064					&env->idmap_scratch);
 9065		}
 9066	}
 9067	return 0;
 9068}
 9069
 9070static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_st,
 9071						 struct bpf_kfunc_call_arg_meta *meta)
 9072{
 9073	int iter_frameno = meta->iter.frameno;
 9074	int iter_spi = meta->iter.spi;
 9075
 9076	return &cur_st->frame[iter_frameno]->stack[iter_spi].spilled_ptr;
 9077}
 9078
 9079/* process_iter_next_call() is called when verifier gets to iterator's next
 9080 * "method" (e.g., bpf_iter_num_next() for numbers iterator) call. We'll refer
 9081 * to it as just "iter_next()" in comments below.
 9082 *
 9083 * BPF verifier relies on a crucial contract for any iter_next()
 9084 * implementation: it should *eventually* return NULL, and once that happens
 9085 * it should keep returning NULL. That is, once iterator exhausts elements to
 9086 * iterate, it should never reset or spuriously return new elements.
 9087 *
 9088 * With the assumption of such contract, process_iter_next_call() simulates
 9089 * a fork in the verifier state to validate loop logic correctness and safety
 9090 * without having to simulate infinite amount of iterations.
 9091 *
 9092 * In current state, we first assume that iter_next() returned NULL and
 9093 * iterator state is set to DRAINED (BPF_ITER_STATE_DRAINED). In such
 9094 * conditions we should not form an infinite loop and should eventually reach
 9095 * exit.
 9096 *
 9097 * Besides that, we also fork current state and enqueue it for later
 9098 * verification. In a forked state we keep iterator state as ACTIVE
 9099 * (BPF_ITER_STATE_ACTIVE) and assume non-NULL return from iter_next(). We
 9100 * also bump iteration depth to prevent erroneous infinite loop detection
 9101 * later on (see iter_active_depths_differ() comment for details). In this
 9102 * state we assume that we'll eventually loop back to another iter_next()
 9103 * calls (it could be in exactly same location or in some other instruction,
 9104 * it doesn't matter, we don't make any unnecessary assumptions about this,
 9105 * everything revolves around iterator state in a stack slot, not which
 9106 * instruction is calling iter_next()). When that happens, we either will come
 9107 * to iter_next() with equivalent state and can conclude that next iteration
 9108 * will proceed in exactly the same way as we just verified, so it's safe to
 9109 * assume that loop converges. If not, we'll go on another iteration
 9110 * simulation with a different input state, until all possible starting states
 9111 * are validated or we reach maximum number of instructions limit.
 9112 *
 9113 * This way, we will either exhaustively discover all possible input states
 9114 * that iterator loop can start with and eventually will converge, or we'll
 9115 * effectively regress into bounded loop simulation logic and either reach
 9116 * maximum number of instructions if loop is not provably convergent, or there
 9117 * is some statically known limit on number of iterations (e.g., if there is
 9118 * an explicit `if n > 100 then break;` statement somewhere in the loop).
 9119 *
 9120 * Iteration convergence logic in is_state_visited() relies on exact
 9121 * states comparison, which ignores read and precision marks.
 9122 * This is necessary because read and precision marks are not finalized
 9123 * while in the loop. Exact comparison might preclude convergence for
 9124 * simple programs like below:
 9125 *
 9126 *     i = 0;
 9127 *     while(iter_next(&it))
 9128 *       i++;
 9129 *
 9130 * At each iteration step i++ would produce a new distinct state and
 9131 * eventually instruction processing limit would be reached.
 9132 *
 9133 * To avoid such behavior speculatively forget (widen) range for
 9134 * imprecise scalar registers, if those registers were not precise at the
 9135 * end of the previous iteration and do not match exactly.
 9136 *
 9137 * This is a conservative heuristic that allows to verify wide range of programs,
 9138 * however it precludes verification of programs that conjure an
 9139 * imprecise value on the first loop iteration and use it as precise on a second.
 9140 * For example, the following safe program would fail to verify:
 9141 *
 9142 *     struct bpf_num_iter it;
 9143 *     int arr[10];
 9144 *     int i = 0, a = 0;
 9145 *     bpf_iter_num_new(&it, 0, 10);
 9146 *     while (bpf_iter_num_next(&it)) {
 9147 *       if (a == 0) {
 9148 *         a = 1;
 9149 *         i = 7; // Because i changed verifier would forget
 9150 *                // it's range on second loop entry.
 9151 *       } else {
 9152 *         arr[i] = 42; // This would fail to verify.
 9153 *       }
 9154 *     }
 9155 *     bpf_iter_num_destroy(&it);
 9156 */
 9157static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx,
 9158				  struct bpf_kfunc_call_arg_meta *meta)
 9159{
 9160	struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;
 9161	struct bpf_func_state *cur_fr = cur_st->frame[cur_st->curframe], *queued_fr;
 9162	struct bpf_reg_state *cur_iter, *queued_iter;
 9163
 9164	BTF_TYPE_EMIT(struct bpf_iter);
 9165
 9166	cur_iter = get_iter_from_state(cur_st, meta);
 9167
 9168	if (cur_iter->iter.state != BPF_ITER_STATE_ACTIVE &&
 9169	    cur_iter->iter.state != BPF_ITER_STATE_DRAINED) {
 9170		verifier_bug(env, "unexpected iterator state %d (%s)",
 9171			     cur_iter->iter.state, iter_state_str(cur_iter->iter.state));
 9172		return -EFAULT;
 9173	}
 9174
 9175	if (cur_iter->iter.state == BPF_ITER_STATE_ACTIVE) {
 9176		/* Because iter_next() call is a checkpoint is_state_visitied()
 9177		 * should guarantee parent state with same call sites and insn_idx.
 9178		 */
 9179		if (!cur_st->parent || cur_st->parent->insn_idx != insn_idx ||
 9180		    !same_callsites(cur_st->parent, cur_st)) {
 9181			verifier_bug(env, "bad parent state for iter next call");
 9182			return -EFAULT;
 9183		}
 9184		/* Note cur_st->parent in the call below, it is necessary to skip
 9185		 * checkpoint created for cur_st by is_state_visited()
 9186		 * right at this instruction.
 9187		 */
 9188		prev_st = find_prev_entry(env, cur_st->parent, insn_idx);
 9189		/* branch out active iter state */
 9190		queued_st = push_stack(env, insn_idx + 1, insn_idx, false);
 9191		if (IS_ERR(queued_st))
 9192			return PTR_ERR(queued_st);
 9193
 9194		queued_iter = get_iter_from_state(queued_st, meta);
 9195		queued_iter->iter.state = BPF_ITER_STATE_ACTIVE;
 9196		queued_iter->iter.depth++;
 9197		if (prev_st)
 9198			widen_imprecise_scalars(env, prev_st, queued_st);
 9199
 9200		queued_fr = queued_st->frame[queued_st->curframe];
 9201		mark_ptr_not_null_reg(&queued_fr->regs[BPF_REG_0]);
 9202	}
 9203
 9204	/* switch to DRAINED state, but keep the depth unchanged */
 9205	/* mark current iter state as drained and assume returned NULL */
 9206	cur_iter->iter.state = BPF_ITER_STATE_DRAINED;
 9207	__mark_reg_const_zero(env, &cur_fr->regs[BPF_REG_0]);
 9208
 9209	return 0;
 9210}
 9211
 9212static bool arg_type_is_mem_size(enum bpf_arg_type type)
 9213{
 9214	return type == ARG_CONST_SIZE ||
 9215	       type == ARG_CONST_SIZE_OR_ZERO;
 9216}
 9217
 9218static bool arg_type_is_raw_mem(enum bpf_arg_type type)
 9219{
 9220	return base_type(type) == ARG_PTR_TO_MEM &&
 9221	       type & MEM_UNINIT;
 9222}
 9223
 9224static bool arg_type_is_release(enum bpf_arg_type type)
 9225{
 9226	return type & OBJ_RELEASE;
 9227}
 9228
 9229static bool arg_type_is_dynptr(enum bpf_arg_type type)
 9230{
 9231	return base_type(type) == ARG_PTR_TO_DYNPTR;
 9232}
 9233
 9234static int resolve_map_arg_type(struct bpf_verifier_env *env,
 9235				 const struct bpf_call_arg_meta *meta,
 9236				 enum bpf_arg_type *arg_type)
 9237{
 9238	if (!meta->map_ptr) {
 9239		/* kernel subsystem misconfigured verifier */
 9240		verifier_bug(env, "invalid map_ptr to access map->type");
 9241		return -EFAULT;
 9242	}
 9243
 9244	switch (meta->map_ptr->map_type) {
 9245	case BPF_MAP_TYPE_SOCKMAP:
 9246	case BPF_MAP_TYPE_SOCKHASH:
 9247		if (*arg_type == ARG_PTR_TO_MAP_VALUE) {
 9248			*arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON;
 9249		} else {
 9250			verbose(env, "invalid arg_type for sockmap/sockhash\n");
 9251			return -EINVAL;
 9252		}
 9253		break;
 9254	case BPF_MAP_TYPE_BLOOM_FILTER:
 9255		if (meta->func_id == BPF_FUNC_map_peek_elem)
 9256			*arg_type = ARG_PTR_TO_MAP_VALUE;
 9257		break;
 9258	default:
 9259		break;
 9260	}
 9261	return 0;
 9262}
 9263
 9264struct bpf_reg_types {
 9265	const enum bpf_reg_type types[10];
 9266	u32 *btf_id;
 9267};
 9268
 9269static const struct bpf_reg_types sock_types = {
 9270	.types = {
 9271		PTR_TO_SOCK_COMMON,
 9272		PTR_TO_SOCKET,
 9273		PTR_TO_TCP_SOCK,
 9274		PTR_TO_XDP_SOCK,
 9275	},
 9276};
 9277
 9278#ifdef CONFIG_NET
 9279static const struct bpf_reg_types btf_id_sock_common_types = {
 9280	.types = {
 9281		PTR_TO_SOCK_COMMON,
 9282		PTR_TO_SOCKET,
 9283		PTR_TO_TCP_SOCK,
 9284		PTR_TO_XDP_SOCK,
 9285		PTR_TO_BTF_ID,
 9286		PTR_TO_BTF_ID | PTR_TRUSTED,
 9287	},
 9288	.btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
 9289};
 9290#endif
 9291
 9292static const struct bpf_reg_types mem_types = {
 9293	.types = {
 9294		PTR_TO_STACK,
 9295		PTR_TO_PACKET,
 9296		PTR_TO_PACKET_META,
 9297		PTR_TO_MAP_KEY,
 9298		PTR_TO_MAP_VALUE,
 9299		PTR_TO_MEM,
 9300		PTR_TO_MEM | MEM_RINGBUF,
 9301		PTR_TO_BUF,
 9302		PTR_TO_BTF_ID | PTR_TRUSTED,
 9303	},
 9304};
 9305
 9306static const struct bpf_reg_types spin_lock_types = {
 9307	.types = {
 9308		PTR_TO_MAP_VALUE,
 9309		PTR_TO_BTF_ID | MEM_ALLOC,
 9310	}
 9311};
 9312
 9313static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } };
 9314static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } };
 9315static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } };
 9316static const struct bpf_reg_types ringbuf_mem_types = { .types = { PTR_TO_MEM | MEM_RINGBUF } };
 9317static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } };
 9318static const struct bpf_reg_types btf_ptr_types = {
 9319	.types = {
 9320		PTR_TO_BTF_ID,
 9321		PTR_TO_BTF_ID | PTR_TRUSTED,
 9322		PTR_TO_BTF_ID | MEM_RCU,
 9323	},
 9324};
 9325static const struct bpf_reg_types percpu_btf_ptr_types = {
 9326	.types = {
 9327		PTR_TO_BTF_ID | MEM_PERCPU,
 9328		PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU,
 9329		PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED,
 9330	}
 9331};
 9332static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } };
 9333static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } };
 9334static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } };
 9335static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } };
 9336static const struct bpf_reg_types kptr_xchg_dest_types = {
 9337	.types = {
 9338		PTR_TO_MAP_VALUE,
 9339		PTR_TO_BTF_ID | MEM_ALLOC
 9340	}
 9341};
 9342static const struct bpf_reg_types dynptr_types = {
 9343	.types = {
 9344		PTR_TO_STACK,
 9345		CONST_PTR_TO_DYNPTR,
 9346	}
 9347};
 9348
 9349static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
 9350	[ARG_PTR_TO_MAP_KEY]		= &mem_types,
 9351	[ARG_PTR_TO_MAP_VALUE]		= &mem_types,
 9352	[ARG_CONST_SIZE]		= &scalar_types,
 9353	[ARG_CONST_SIZE_OR_ZERO]	= &scalar_types,
 9354	[ARG_CONST_ALLOC_SIZE_OR_ZERO]	= &scalar_types,
 9355	[ARG_CONST_MAP_PTR]		= &const_map_ptr_types,
 9356	[ARG_PTR_TO_CTX]		= &context_types,
 9357	[ARG_PTR_TO_SOCK_COMMON]	= &sock_types,
 9358#ifdef CONFIG_NET
 9359	[ARG_PTR_TO_BTF_ID_SOCK_COMMON]	= &btf_id_sock_common_types,
 9360#endif
 9361	[ARG_PTR_TO_SOCKET]		= &fullsock_types,
 9362	[ARG_PTR_TO_BTF_ID]		= &btf_ptr_types,
 9363	[ARG_PTR_TO_SPIN_LOCK]		= &spin_lock_types,
 9364	[ARG_PTR_TO_MEM]		= &mem_types,
 9365	[ARG_PTR_TO_RINGBUF_MEM]	= &ringbuf_mem_types,
 9366	[ARG_PTR_TO_PERCPU_BTF_ID]	= &percpu_btf_ptr_types,
 9367	[ARG_PTR_TO_FUNC]		= &func_ptr_types,
 9368	[ARG_PTR_TO_STACK]		= &stack_ptr_types,
 9369	[ARG_PTR_TO_CONST_STR]		= &const_str_ptr_types,
 9370	[ARG_PTR_TO_TIMER]		= &timer_types,
 9371	[ARG_KPTR_XCHG_DEST]		= &kptr_xchg_dest_types,
 9372	[ARG_PTR_TO_DYNPTR]		= &dynptr_types,
 9373};
 9374
 9375static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
 9376			  enum bpf_arg_type arg_type,
 9377			  const u32 *arg_btf_id,
 9378			  struct bpf_call_arg_meta *meta)
 9379{
 9380	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
 9381	enum bpf_reg_type expected, type = reg->type;
 9382	const struct bpf_reg_types *compatible;
 9383	int i, j;
 9384
 9385	compatible = compatible_reg_types[base_type(arg_type)];
 9386	if (!compatible) {
 9387		verifier_bug(env, "unsupported arg type %d", arg_type);
 9388		return -EFAULT;
 9389	}
 9390
 9391	/* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY,
 9392	 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY
 9393	 *
 9394	 * Same for MAYBE_NULL:
 9395	 *
 9396	 * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL,
 9397	 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL
 9398	 *
 9399	 * ARG_PTR_TO_MEM is compatible with PTR_TO_MEM that is tagged with a dynptr type.
 9400	 *
 9401	 * Therefore we fold these flags depending on the arg_type before comparison.
 9402	 */
 9403	if (arg_type & MEM_RDONLY)
 9404		type &= ~MEM_RDONLY;
 9405	if (arg_type & PTR_MAYBE_NULL)
 9406		type &= ~PTR_MAYBE_NULL;
 9407	if (base_type(arg_type) == ARG_PTR_TO_MEM)
 9408		type &= ~DYNPTR_TYPE_FLAG_MASK;
 9409
 9410	/* Local kptr types are allowed as the source argument of bpf_kptr_xchg */
 9411	if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type) && regno == BPF_REG_2) {
 9412		type &= ~MEM_ALLOC;
 9413		type &= ~MEM_PERCPU;
 9414	}
 9415
 9416	for (i = 0; i < ARRAY_SIZE(compatible->types); i++) {
 9417		expected = compatible->types[i];
 9418		if (expected == NOT_INIT)
 9419			break;
 9420
 9421		if (type == expected)
 9422			goto found;
 9423	}
 9424
 9425	verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type));
 9426	for (j = 0; j + 1 < i; j++)
 9427		verbose(env, "%s, ", reg_type_str(env, compatible->types[j]));
 9428	verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));
 9429	return -EACCES;
 9430
 9431found:
 9432	if (base_type(reg->type) != PTR_TO_BTF_ID)
 9433		return 0;
 9434
 9435	if (compatible == &mem_types) {
 9436		if (!(arg_type & MEM_RDONLY)) {
 9437			verbose(env,
 9438				"%s() may write into memory pointed by R%d type=%s\n",
 9439				func_id_name(meta->func_id),
 9440				regno, reg_type_str(env, reg->type));
 9441			return -EACCES;
 9442		}
 9443		return 0;
 9444	}
 9445
 9446	switch ((int)reg->type) {
 9447	case PTR_TO_BTF_ID:
 9448	case PTR_TO_BTF_ID | PTR_TRUSTED:
 9449	case PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL:
 9450	case PTR_TO_BTF_ID | MEM_RCU:
 9451	case PTR_TO_BTF_ID | PTR_MAYBE_NULL:
 9452	case PTR_TO_BTF_ID | PTR_MAYBE_NULL | MEM_RCU:
 9453	{
 9454		/* For bpf_sk_release, it needs to match against first member
 9455		 * 'struct sock_common', hence make an exception for it. This
 9456		 * allows bpf_sk_release to work for multiple socket types.
 9457		 */
 9458		bool strict_type_match = arg_type_is_release(arg_type) &&
 9459					 meta->func_id != BPF_FUNC_sk_release;
 9460
 9461		if (type_may_be_null(reg->type) &&
 9462		    (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) {
 9463			verbose(env, "Possibly NULL pointer passed to helper arg%d\n", regno);
 9464			return -EACCES;
 9465		}
 9466
 9467		if (!arg_btf_id) {
 9468			if (!compatible->btf_id) {
 9469				verifier_bug(env, "missing arg compatible BTF ID");
 9470				return -EFAULT;
 9471			}
 9472			arg_btf_id = compatible->btf_id;
 9473		}
 9474
 9475		if (meta->func_id == BPF_FUNC_kptr_xchg) {
 9476			if (map_kptr_match_type(env, meta->kptr_field, reg, regno))
 9477				return -EACCES;
 9478		} else {
 9479			if (arg_btf_id == BPF_PTR_POISON) {
 9480				verbose(env, "verifier internal error:");
 9481				verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n",
 9482					regno);
 9483				return -EACCES;
 9484			}
 9485
 9486			if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
 9487						  btf_vmlinux, *arg_btf_id,
 9488						  strict_type_match)) {
 9489				verbose(env, "R%d is of type %s but %s is expected\n",
 9490					regno, btf_type_name(reg->btf, reg->btf_id),
 9491					btf_type_name(btf_vmlinux, *arg_btf_id));
 9492				return -EACCES;
 9493			}
 9494		}
 9495		break;
 9496	}
 9497	case PTR_TO_BTF_ID | MEM_ALLOC:
 9498	case PTR_TO_BTF_ID | MEM_PERCPU | MEM_ALLOC:
 9499		if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock &&
 9500		    meta->func_id != BPF_FUNC_kptr_xchg) {
 9501			verifier_bug(env, "unimplemented handling of MEM_ALLOC");
 9502			return -EFAULT;
 9503		}
 9504		/* Check if local kptr in src arg matches kptr in dst arg */
 9505		if (meta->func_id == BPF_FUNC_kptr_xchg && regno == BPF_REG_2) {
 9506			if (map_kptr_match_type(env, meta->kptr_field, reg, regno))
 9507				return -EACCES;
 9508		}
 9509		break;
 9510	case PTR_TO_BTF_ID | MEM_PERCPU:
 9511	case PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU:
 9512	case PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED:
 9513		/* Handled by helper specific checks */
 9514		break;
 9515	default:
 9516		verifier_bug(env, "invalid PTR_TO_BTF_ID register for type match");
 9517		return -EFAULT;
 9518	}
 9519	return 0;
 9520}
 9521
 9522static struct btf_field *
 9523reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields)
 9524{
 9525	struct btf_field *field;
 9526	struct btf_record *rec;
 9527
 9528	rec = reg_btf_record(reg);
 9529	if (!rec)
 9530		return NULL;
 9531
 9532	field = btf_record_find(rec, off, fields);
 9533	if (!field)
 9534		return NULL;
 9535
 9536	return field;
 9537}
 9538
 9539static int check_func_arg_reg_off(struct bpf_verifier_env *env,
 9540				  const struct bpf_reg_state *reg, int regno,
 9541				  enum bpf_arg_type arg_type)
 9542{
 9543	u32 type = reg->type;
 9544
 9545	/* When referenced register is passed to release function, its fixed
 9546	 * offset must be 0.
 9547	 *
 9548	 * We will check arg_type_is_release reg has ref_obj_id when storing
 9549	 * meta->release_regno.
 9550	 */
 9551	if (arg_type_is_release(arg_type)) {
 9552		/* ARG_PTR_TO_DYNPTR with OBJ_RELEASE is a bit special, as it
 9553		 * may not directly point to the object being released, but to
 9554		 * dynptr pointing to such object, which might be at some offset
 9555		 * on the stack. In that case, we simply to fallback to the
 9556		 * default handling.
 9557		 */
 9558		if (arg_type_is_dynptr(arg_type) && type == PTR_TO_STACK)
 9559			return 0;
 9560
 9561		/* Doing check_ptr_off_reg check for the offset will catch this
 9562		 * because fixed_off_ok is false, but checking here allows us
 9563		 * to give the user a better error message.
 9564		 */
 9565		if (reg->off) {
 9566			verbose(env, "R%d must have zero offset when passed to release func or trusted arg to kfunc\n",
 9567				regno);
 9568			return -EINVAL;
 9569		}
 9570		return __check_ptr_off_reg(env, reg, regno, false);
 9571	}
 9572
 9573	switch (type) {
 9574	/* Pointer types where both fixed and variable offset is explicitly allowed: */
 9575	case PTR_TO_STACK:
 9576	case PTR_TO_PACKET:
 9577	case PTR_TO_PACKET_META:
 9578	case PTR_TO_MAP_KEY:
 9579	case PTR_TO_MAP_VALUE:
 9580	case PTR_TO_MEM:
 9581	case PTR_TO_MEM | MEM_RDONLY:
 9582	case PTR_TO_MEM | MEM_RINGBUF:
 9583	case PTR_TO_BUF:
 9584	case PTR_TO_BUF | MEM_RDONLY:
 9585	case PTR_TO_ARENA:
 9586	case SCALAR_VALUE:
 9587		return 0;
 9588	/* All the rest must be rejected, except PTR_TO_BTF_ID which allows
 9589	 * fixed offset.
 9590	 */
 9591	case PTR_TO_BTF_ID:
 9592	case PTR_TO_BTF_ID | MEM_ALLOC:
 9593	case PTR_TO_BTF_ID | PTR_TRUSTED:
 9594	case PTR_TO_BTF_ID | MEM_RCU:
 9595	case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF:
 9596	case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU:
 9597		/* When referenced PTR_TO_BTF_ID is passed to release function,
 9598		 * its fixed offset must be 0. In the other cases, fixed offset
 9599		 * can be non-zero. This was already checked above. So pass
 9600		 * fixed_off_ok as true to allow fixed offset for all other
 9601		 * cases. var_off always must be 0 for PTR_TO_BTF_ID, hence we
 9602		 * still need to do checks instead of returning.
 9603		 */
 9604		return __check_ptr_off_reg(env, reg, regno, true);
 9605	default:
 9606		return __check_ptr_off_reg(env, reg, regno, false);
 9607	}
 9608}
 9609
 9610static struct bpf_reg_state *get_dynptr_arg_reg(struct bpf_verifier_env *env,
 9611						const struct bpf_func_proto *fn,
 9612						struct bpf_reg_state *regs)
 9613{
 9614	struct bpf_reg_state *state = NULL;
 9615	int i;
 9616
 9617	for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++)
 9618		if (arg_type_is_dynptr(fn->arg_type[i])) {
 9619			if (state) {
 9620				verbose(env, "verifier internal error: multiple dynptr args\n");
 9621				return NULL;
 9622			}
 9623			state = &regs[BPF_REG_1 + i];
 9624		}
 9625
 9626	if (!state)
 9627		verbose(env, "verifier internal error: no dynptr arg found\n");
 9628
 9629	return state;
 9630}
 9631
 9632static int dynptr_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
 9633{
 9634	struct bpf_func_state *state = func(env, reg);
 9635	int spi;
 9636
 9637	if (reg->type == CONST_PTR_TO_DYNPTR)
 9638		return reg->id;
 9639	spi = dynptr_get_spi(env, reg);
 9640	if (spi < 0)
 9641		return spi;
 9642	return state->stack[spi].spilled_ptr.id;
 9643}
 9644
 9645static int dynptr_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
 9646{
 9647	struct bpf_func_state *state = func(env, reg);
 9648	int spi;
 9649
 9650	if (reg->type == CONST_PTR_TO_DYNPTR)
 9651		return reg->ref_obj_id;
 9652	spi = dynptr_get_spi(env, reg);
 9653	if (spi < 0)
 9654		return spi;
 9655	return state->stack[spi].spilled_ptr.ref_obj_id;
 9656}
 9657
 9658static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env,
 9659					    struct bpf_reg_state *reg)
 9660{
 9661	struct bpf_func_state *state = func(env, reg);
 9662	int spi;
 9663
 9664	if (reg->type == CONST_PTR_TO_DYNPTR)
 9665		return reg->dynptr.type;
 9666
 9667	spi = __get_spi(reg->off);
 9668	if (spi < 0) {
 9669		verbose(env, "verifier internal error: invalid spi when querying dynptr type\n");
 9670		return BPF_DYNPTR_TYPE_INVALID;
 9671	}
 9672
 9673	return state->stack[spi].spilled_ptr.dynptr.type;
 9674}
 9675
 9676static int check_reg_const_str(struct bpf_verifier_env *env,
 9677			       struct bpf_reg_state *reg, u32 regno)
 9678{
 9679	struct bpf_map *map = reg->map_ptr;
 9680	int err;
 9681	int map_off;
 9682	u64 map_addr;
 9683	char *str_ptr;
 9684
 9685	if (reg->type != PTR_TO_MAP_VALUE)
 9686		return -EINVAL;
 9687
 9688	if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) {
 9689		verbose(env, "R%d points to insn_array map which cannot be used as const string\n", regno);
 9690		return -EACCES;
 9691	}
 9692
 9693	if (!bpf_map_is_rdonly(map)) {
 9694		verbose(env, "R%d does not point to a readonly map'\n", regno);
 9695		return -EACCES;
 9696	}
 9697
 9698	if (!tnum_is_const(reg->var_off)) {
 9699		verbose(env, "R%d is not a constant address'\n", regno);
 9700		return -EACCES;
 9701	}
 9702
 9703	if (!map->ops->map_direct_value_addr) {
 9704		verbose(env, "no direct value access support for this map type\n");
 9705		return -EACCES;
 9706	}
 9707
 9708	err = check_map_access(env, regno, reg->off,
 9709			       map->value_size - reg->off, false,
 9710			       ACCESS_HELPER);
 9711	if (err)
 9712		return err;
 9713
 9714	map_off = reg->off + reg->var_off.value;
 9715	err = map->ops->map_direct_value_addr(map, &map_addr, map_off);
 9716	if (err) {
 9717		verbose(env, "direct value access on string failed\n");
 9718		return err;
 9719	}
 9720
 9721	str_ptr = (char *)(long)(map_addr);
 9722	if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) {
 9723		verbose(env, "string is not zero-terminated\n");
 9724		return -EINVAL;
 9725	}
 9726	return 0;
 9727}
 9728
 9729/* Returns constant key value in `value` if possible, else negative error */
 9730static int get_constant_map_key(struct bpf_verifier_env *env,
 9731				struct bpf_reg_state *key,
 9732				u32 key_size,
 9733				s64 *value)
 9734{
 9735	struct bpf_func_state *state = func(env, key);
 9736	struct bpf_reg_state *reg;
 9737	int slot, spi, off;
 9738	int spill_size = 0;
 9739	int zero_size = 0;
 9740	int stack_off;
 9741	int i, err;
 9742	u8 *stype;
 9743
 9744	if (!env->bpf_capable)
 9745		return -EOPNOTSUPP;
 9746	if (key->type != PTR_TO_STACK)
 9747		return -EOPNOTSUPP;
 9748	if (!tnum_is_const(key->var_off))
 9749		return -EOPNOTSUPP;
 9750
 9751	stack_off = key->off + key->var_off.value;
 9752	slot = -stack_off - 1;
 9753	spi = slot / BPF_REG_SIZE;
 9754	off = slot % BPF_REG_SIZE;
 9755	stype = state->stack[spi].slot_type;
 9756
 9757	/* First handle precisely tracked STACK_ZERO */
 9758	for (i = off; i >= 0 && stype[i] == STACK_ZERO; i--)
 9759		zero_size++;
 9760	if (zero_size >= key_size) {
 9761		*value = 0;
 9762		return 0;
 9763	}
 9764
 9765	/* Check that stack contains a scalar spill of expected size */
 9766	if (!is_spilled_scalar_reg(&state->stack[spi]))
 9767		return -EOPNOTSUPP;
 9768	for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--)
 9769		spill_size++;
 9770	if (spill_size != key_size)
 9771		return -EOPNOTSUPP;
 9772
 9773	reg = &state->stack[spi].spilled_ptr;
 9774	if (!tnum_is_const(reg->var_off))
 9775		/* Stack value not statically known */
 9776		return -EOPNOTSUPP;
 9777
 9778	/* We are relying on a constant value. So mark as precise
 9779	 * to prevent pruning on it.
 9780	 */
 9781	bt_set_frame_slot(&env->bt, key->frameno, spi);
 9782	err = mark_chain_precision_batch(env, env->cur_state);
 9783	if (err < 0)
 9784		return err;
 9785
 9786	*value = reg->var_off.value;
 9787	return 0;
 9788}
 9789
 9790static bool can_elide_value_nullness(enum bpf_map_type type);
 9791
 9792static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 9793			  struct bpf_call_arg_meta *meta,
 9794			  const struct bpf_func_proto *fn,
 9795			  int insn_idx)
 9796{
 9797	u32 regno = BPF_REG_1 + arg;
 9798	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
 9799	enum bpf_arg_type arg_type = fn->arg_type[arg];
 9800	enum bpf_reg_type type = reg->type;
 9801	u32 *arg_btf_id = NULL;
 9802	u32 key_size;
 9803	int err = 0;
 9804
 9805	if (arg_type == ARG_DONTCARE)
 9806		return 0;
 9807
 9808	err = check_reg_arg(env, regno, SRC_OP);
 9809	if (err)
 9810		return err;
 9811
 9812	if (arg_type == ARG_ANYTHING) {
 9813		if (is_pointer_value(env, regno)) {
 9814			verbose(env, "R%d leaks addr into helper function\n",
 9815				regno);
 9816			return -EACCES;
 9817		}
 9818		return 0;
 9819	}
 9820
 9821	if (type_is_pkt_pointer(type) &&
 9822	    !may_access_direct_pkt_data(env, meta, BPF_READ)) {
 9823		verbose(env, "helper access to the packet is not allowed\n");
 9824		return -EACCES;
 9825	}
 9826
 9827	if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) {
 9828		err = resolve_map_arg_type(env, meta, &arg_type);
 9829		if (err)
 9830			return err;
 9831	}
 9832
 9833	if (register_is_null(reg) && type_may_be_null(arg_type))
 9834		/* A NULL register has a SCALAR_VALUE type, so skip
 9835		 * type checking.
 9836		 */
 9837		goto skip_type_check;
 9838
 9839	/* arg_btf_id and arg_size are in a union. */
 9840	if (base_type(arg_type) == ARG_PTR_TO_BTF_ID ||
 9841	    base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK)
 9842		arg_btf_id = fn->arg_btf_id[arg];
 9843
 9844	err = check_reg_type(env, regno, arg_type, arg_btf_id, meta);
 9845	if (err)
 9846		return err;
 9847
 9848	err = check_func_arg_reg_off(env, reg, regno, arg_type);
 9849	if (err)
 9850		return err;
 9851
 9852skip_type_check:
 9853	if (arg_type_is_release(arg_type)) {
 9854		if (arg_type_is_dynptr(arg_type)) {
 9855			struct bpf_func_state *state = func(env, reg);
 9856			int spi;
 9857
 9858			/* Only dynptr created on stack can be released, thus
 9859			 * the get_spi and stack state checks for spilled_ptr
 9860			 * should only be done before process_dynptr_func for
 9861			 * PTR_TO_STACK.
 9862			 */
 9863			if (reg->type == PTR_TO_STACK) {
 9864				spi = dynptr_get_spi(env, reg);
 9865				if (spi < 0 || !state->stack[spi].spilled_ptr.ref_obj_id) {
 9866					verbose(env, "arg %d is an unacquired reference\n", regno);
 9867					return -EINVAL;
 9868				}
 9869			} else {
 9870				verbose(env, "cannot release unowned const bpf_dynptr\n");
 9871				return -EINVAL;
 9872			}
 9873		} else if (!reg->ref_obj_id && !register_is_null(reg)) {
 9874			verbose(env, "R%d must be referenced when passed to release function\n",
 9875				regno);
 9876			return -EINVAL;
 9877		}
 9878		if (meta->release_regno) {
 9879			verifier_bug(env, "more than one release argument");
 9880			return -EFAULT;
 9881		}
 9882		meta->release_regno = regno;
 9883	}
 9884
 9885	if (reg->ref_obj_id && base_type(arg_type) != ARG_KPTR_XCHG_DEST) {
 9886		if (meta->ref_obj_id) {
 9887			verbose(env, "more than one arg with ref_obj_id R%d %u %u",
 9888				regno, reg->ref_obj_id,
 9889				meta->ref_obj_id);
 9890			return -EACCES;
 9891		}
 9892		meta->ref_obj_id = reg->ref_obj_id;
 9893	}
 9894
 9895	switch (base_type(arg_type)) {
 9896	case ARG_CONST_MAP_PTR:
 9897		/* bpf_map_xxx(map_ptr) call: remember that map_ptr */
 9898		if (meta->map_ptr) {
 9899			/* Use map_uid (which is unique id of inner map) to reject:
 9900			 * inner_map1 = bpf_map_lookup_elem(outer_map, key1)
 9901			 * inner_map2 = bpf_map_lookup_elem(outer_map, key2)
 9902			 * if (inner_map1 && inner_map2) {
 9903			 *     timer = bpf_map_lookup_elem(inner_map1);
 9904			 *     if (timer)
 9905			 *         // mismatch would have been allowed
 9906			 *         bpf_timer_init(timer, inner_map2);
 9907			 * }
 9908			 *
 9909			 * Comparing map_ptr is enough to distinguish normal and outer maps.
 9910			 */
 9911			if (meta->map_ptr != reg->map_ptr ||
 9912			    meta->map_uid != reg->map_uid) {
 9913				verbose(env,
 9914					"timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n",
 9915					meta->map_uid, reg->map_uid);
 9916				return -EINVAL;
 9917			}
 9918		}
 9919		meta->map_ptr = reg->map_ptr;
 9920		meta->map_uid = reg->map_uid;
 9921		break;
 9922	case ARG_PTR_TO_MAP_KEY:
 9923		/* bpf_map_xxx(..., map_ptr, ..., key) call:
 9924		 * check that [key, key + map->key_size) are within
 9925		 * stack limits and initialized
 9926		 */
 9927		if (!meta->map_ptr) {
 9928			/* in function declaration map_ptr must come before
 9929			 * map_key, so that it's verified and known before
 9930			 * we have to check map_key here. Otherwise it means
 9931			 * that kernel subsystem misconfigured verifier
 9932			 */
 9933			verifier_bug(env, "invalid map_ptr to access map->key");
 9934			return -EFAULT;
 9935		}
 9936		key_size = meta->map_ptr->key_size;
 9937		err = check_helper_mem_access(env, regno, key_size, BPF_READ, false, NULL);
 9938		if (err)
 9939			return err;
 9940		if (can_elide_value_nullness(meta->map_ptr->map_type)) {
 9941			err = get_constant_map_key(env, reg, key_size, &meta->const_map_key);
 9942			if (err < 0) {
 9943				meta->const_map_key = -1;
 9944				if (err == -EOPNOTSUPP)
 9945					err = 0;
 9946				else
 9947					return err;
 9948			}
 9949		}
 9950		break;
 9951	case ARG_PTR_TO_MAP_VALUE:
 9952		if (type_may_be_null(arg_type) && register_is_null(reg))
 9953			return 0;
 9954
 9955		/* bpf_map_xxx(..., map_ptr, ..., value) call:
 9956		 * check [value, value + map->value_size) validity
 9957		 */
 9958		if (!meta->map_ptr) {
 9959			/* kernel subsystem misconfigured verifier */
 9960			verifier_bug(env, "invalid map_ptr to access map->value");
 9961			return -EFAULT;
 9962		}
 9963		meta->raw_mode = arg_type & MEM_UNINIT;
 9964		err = check_helper_mem_access(env, regno, meta->map_ptr->value_size,
 9965					      arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ,
 9966					      false, meta);
 9967		break;
 9968	case ARG_PTR_TO_PERCPU_BTF_ID:
 9969		if (!reg->btf_id) {
 9970			verbose(env, "Helper has invalid btf_id in R%d\n", regno);
 9971			return -EACCES;
 9972		}
 9973		meta->ret_btf = reg->btf;
 9974		meta->ret_btf_id = reg->btf_id;
 9975		break;
 9976	case ARG_PTR_TO_SPIN_LOCK:
 9977		if (in_rbtree_lock_required_cb(env)) {
 9978			verbose(env, "can't spin_{lock,unlock} in rbtree cb\n");
 9979			return -EACCES;
 9980		}
 9981		if (meta->func_id == BPF_FUNC_spin_lock) {
 9982			err = process_spin_lock(env, regno, PROCESS_SPIN_LOCK);
 9983			if (err)
 9984				return err;
 9985		} else if (meta->func_id == BPF_FUNC_spin_unlock) {
 9986			err = process_spin_lock(env, regno, 0);
 9987			if (err)
 9988				return err;
 9989		} else {
 9990			verifier_bug(env, "spin lock arg on unexpected helper");
 9991			return -EFAULT;
 9992		}
 9993		break;
 9994	case ARG_PTR_TO_TIMER:
 9995		err = process_timer_func(env, regno, meta);
 9996		if (err)
 9997			return err;
 9998		break;
 9999	case ARG_PTR_TO_FUNC:
10000		meta->subprogno = reg->subprogno;
10001		break;
10002	case ARG_PTR_TO_MEM:
10003		/* The access to this pointer is only checked when we hit the
10004		 * next is_mem_size argument below.
10005		 */
10006		meta->raw_mode = arg_type & MEM_UNINIT;
10007		if (arg_type & MEM_FIXED_SIZE) {
10008			err = check_helper_mem_access(env, regno, fn->arg_size[arg],
10009						      arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ,
10010						      false, meta);
10011			if (err)
10012				return err;
10013			if (arg_type & MEM_ALIGNED)
10014				err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true);
10015		}
10016		break;
10017	case ARG_CONST_SIZE:
10018		err = check_mem_size_reg(env, reg, regno,
10019					 fn->arg_type[arg - 1] & MEM_WRITE ?
10020					 BPF_WRITE : BPF_READ,
10021					 false, meta);
10022		break;
10023	case ARG_CONST_SIZE_OR_ZERO:
10024		err = check_mem_size_reg(env, reg, regno,
10025					 fn->arg_type[arg - 1] & MEM_WRITE ?
10026					 BPF_WRITE : BPF_READ,
10027					 true, meta);
10028		break;
10029	case ARG_PTR_TO_DYNPTR:
10030		err = process_dynptr_func(env, regno, insn_idx, arg_type, 0);
10031		if (err)
10032			return err;
10033		break;
10034	case ARG_CONST_ALLOC_SIZE_OR_ZERO:
10035		if (!tnum_is_const(reg->var_off)) {
10036			verbose(env, "R%d is not a known constant'\n",
10037				regno);
10038			return -EACCES;
10039		}
10040		meta->mem_size = reg->var_off.value;
10041		err = mark_chain_precision(env, regno);
10042		if (err)
10043			return err;
10044		break;
10045	case ARG_PTR_TO_CONST_STR:
10046	{
10047		err = check_reg_const_str(env, reg, regno);
10048		if (err)
10049			return err;
10050		break;
10051	}
10052	case ARG_KPTR_XCHG_DEST:
10053		err = process_kptr_func(env, regno, meta);
10054		if (err)
10055			return err;
10056		break;
10057	}
10058
10059	return err;
10060}
10061
10062static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
10063{
10064	enum bpf_attach_type eatype = env->prog->expected_attach_type;
10065	enum bpf_prog_type type = resolve_prog_type(env->prog);
10066
10067	if (func_id != BPF_FUNC_map_update_elem &&
10068	    func_id != BPF_FUNC_map_delete_elem)
10069		return false;
10070
10071	/* It's not possible to get access to a locked struct sock in these
10072	 * contexts, so updating is safe.
10073	 */
10074	switch (type) {
10075	case BPF_PROG_TYPE_TRACING:
10076		if (eatype == BPF_TRACE_ITER)
10077			return true;
10078		break;
10079	case BPF_PROG_TYPE_SOCK_OPS:
10080		/* map_update allowed only via dedicated helpers with event type checks */
10081		if (func_id == BPF_FUNC_map_delete_elem)
10082			return true;
10083		break;
10084	case BPF_PROG_TYPE_SOCKET_FILTER:
10085	case BPF_PROG_TYPE_SCHED_CLS:
10086	case BPF_PROG_TYPE_SCHED_ACT:
10087	case BPF_PROG_TYPE_XDP:
10088	case BPF_PROG_TYPE_SK_REUSEPORT:
10089	case BPF_PROG_TYPE_FLOW_DISSECTOR:
10090	case BPF_PROG_TYPE_SK_LOOKUP:
10091		return true;
10092	default:
10093		break;
10094	}
10095
10096	verbose(env, "cannot update sockmap in this context\n");
10097	return false;
10098}
10099
10100static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env)
10101{
10102	return env->prog->jit_requested &&
10103	       bpf_jit_supports_subprog_tailcalls();
10104}
10105
10106static int check_map_func_compatibility(struct bpf_verifier_env *env,
10107					struct bpf_map *map, int func_id)
10108{
10109	if (!map)
10110		return 0;
10111
10112	/* We need a two way check, first is from map perspective ... */
10113	switch (map->map_type) {
10114	case BPF_MAP_TYPE_PROG_ARRAY:
10115		if (func_id != BPF_FUNC_tail_call)
10116			goto error;
10117		break;
10118	case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
10119		if (func_id != BPF_FUNC_perf_event_read &&
10120		    func_id != BPF_FUNC_perf_event_output &&
10121		    func_id != BPF_FUNC_skb_output &&
10122		    func_id != BPF_FUNC_perf_event_read_value &&
10123		    func_id != BPF_FUNC_xdp_output)
10124			goto error;
10125		break;
10126	case BPF_MAP_TYPE_RINGBUF:
10127		if (func_id != BPF_FUNC_ringbuf_output &&
10128		    func_id != BPF_FUNC_ringbuf_reserve &&
10129		    func_id != BPF_FUNC_ringbuf_query &&
10130		    func_id != BPF_FUNC_ringbuf_reserve_dynptr &&
10131		    func_id != BPF_FUNC_ringbuf_submit_dynptr &&
10132		    func_id != BPF_FUNC_ringbuf_discard_dynptr)
10133			goto error;
10134		break;
10135	case BPF_MAP_TYPE_USER_RINGBUF:
10136		if (func_id != BPF_FUNC_user_ringbuf_drain)
10137			goto error;
10138		break;
10139	case BPF_MAP_TYPE_STACK_TRACE:
10140		if (func_id != BPF_FUNC_get_stackid)
10141			goto error;
10142		break;
10143	case BPF_MAP_TYPE_CGROUP_ARRAY:
10144		if (func_id != BPF_FUNC_skb_under_cgroup &&
10145		    func_id != BPF_FUNC_current_task_under_cgroup)
10146			goto error;
10147		break;
10148	case BPF_MAP_TYPE_CGROUP_STORAGE:
10149	case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
10150		if (func_id != BPF_FUNC_get_local_storage)
10151			goto error;
10152		break;
10153	case BPF_MAP_TYPE_DEVMAP:
10154	case BPF_MAP_TYPE_DEVMAP_HASH:
10155		if (func_id != BPF_FUNC_redirect_map &&
10156		    func_id != BPF_FUNC_map_lookup_elem)
10157			goto error;
10158		break;
10159	/* Restrict bpf side of cpumap and xskmap, open when use-cases
10160	 * appear.
10161	 */
10162	case BPF_MAP_TYPE_CPUMAP:
10163		if (func_id != BPF_FUNC_redirect_map)
10164			goto error;
10165		break;
10166	case BPF_MAP_TYPE_XSKMAP:
10167		if (func_id != BPF_FUNC_redirect_map &&
10168		    func_id != BPF_FUNC_map_lookup_elem)
10169			goto error;
10170		break;
10171	case BPF_MAP_TYPE_ARRAY_OF_MAPS:
10172	case BPF_MAP_TYPE_HASH_OF_MAPS:
10173		if (func_id != BPF_FUNC_map_lookup_elem)
10174			goto error;
10175		break;
10176	case BPF_MAP_TYPE_SOCKMAP:
10177		if (func_id != BPF_FUNC_sk_redirect_map &&
10178		    func_id != BPF_FUNC_sock_map_update &&
10179		    func_id != BPF_FUNC_msg_redirect_map &&
10180		    func_id != BPF_FUNC_sk_select_reuseport &&
10181		    func_id != BPF_FUNC_map_lookup_elem &&
10182		    !may_update_sockmap(env, func_id))
10183			goto error;
10184		break;
10185	case BPF_MAP_TYPE_SOCKHASH:
10186		if (func_id != BPF_FUNC_sk_redirect_hash &&
10187		    func_id != BPF_FUNC_sock_hash_update &&
10188		    func_id != BPF_FUNC_msg_redirect_hash &&
10189		    func_id != BPF_FUNC_sk_select_reuseport &&
10190		    func_id != BPF_FUNC_map_lookup_elem &&
10191		    !may_update_sockmap(env, func_id))
10192			goto error;
10193		break;
10194	case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
10195		if (func_id != BPF_FUNC_sk_select_reuseport)
10196			goto error;
10197		break;
10198	case BPF_MAP_TYPE_QUEUE:
10199	case BPF_MAP_TYPE_STACK:
10200		if (func_id != BPF_FUNC_map_peek_elem &&
10201		    func_id != BPF_FUNC_map_pop_elem &&
10202		    func_id != BPF_FUNC_map_push_elem)
10203			goto error;
10204		break;
10205	case BPF_MAP_TYPE_SK_STORAGE:
10206		if (func_id != BPF_FUNC_sk_storage_get &&
10207		    func_id != BPF_FUNC_sk_storage_delete &&
10208		    func_id != BPF_FUNC_kptr_xchg)
10209			goto error;
10210		break;
10211	case BPF_MAP_TYPE_INODE_STORAGE:
10212		if (func_id != BPF_FUNC_inode_storage_get &&
10213		    func_id != BPF_FUNC_inode_storage_delete &&
10214		    func_id != BPF_FUNC_kptr_xchg)
10215			goto error;
10216		break;
10217	case BPF_MAP_TYPE_TASK_STORAGE:
10218		if (func_id != BPF_FUNC_task_storage_get &&
10219		    func_id != BPF_FUNC_task_storage_delete &&
10220		    func_id != BPF_FUNC_kptr_xchg)
10221			goto error;
10222		break;
10223	case BPF_MAP_TYPE_CGRP_STORAGE:
10224		if (func_id != BPF_FUNC_cgrp_storage_get &&
10225		    func_id != BPF_FUNC_cgrp_storage_delete &&
10226		    func_id != BPF_FUNC_kptr_xchg)
10227			goto error;
10228		break;
10229	case BPF_MAP_TYPE_BLOOM_FILTER:
10230		if (func_id != BPF_FUNC_map_peek_elem &&
10231		    func_id != BPF_FUNC_map_push_elem)
10232			goto error;
10233		break;
10234	case BPF_MAP_TYPE_INSN_ARRAY:
10235		goto error;
10236	default:
10237		break;
10238	}
10239
10240	/* ... and second from the function itself. */
10241	switch (func_id) {
10242	case BPF_FUNC_tail_call:
10243		if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
10244			goto error;
10245		if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) {
10246			verbose(env, "mixing of tail_calls and bpf-to-bpf calls is not supported\n");
10247			return -EINVAL;
10248		}
10249		break;
10250	case BPF_FUNC_perf_event_read:
10251	case BPF_FUNC_perf_event_output:
10252	case BPF_FUNC_perf_event_read_value:
10253	case BPF_FUNC_skb_output:
10254	case BPF_FUNC_xdp_output:
10255		if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
10256			goto error;
10257		break;
10258	case BPF_FUNC_ringbuf_output:
10259	case BPF_FUNC_ringbuf_reserve:
10260	case BPF_FUNC_ringbuf_query:
10261	case BPF_FUNC_ringbuf_reserve_dynptr:
10262	case BPF_FUNC_ringbuf_submit_dynptr:
10263	case BPF_FUNC_ringbuf_discard_dynptr:
10264		if (map->map_type != BPF_MAP_TYPE_RINGBUF)
10265			goto error;
10266		break;
10267	case BPF_FUNC_user_ringbuf_drain:
10268		if (map->map_type != BPF_MAP_TYPE_USER_RINGBUF)
10269			goto error;
10270		break;
10271	case BPF_FUNC_get_stackid:
10272		if (map->map_type != BPF_MAP_TYPE_STACK_TRACE)
10273			goto error;
10274		break;
10275	case BPF_FUNC_current_task_under_cgroup:
10276	case BPF_FUNC_skb_under_cgroup:
10277		if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY)
10278			goto error;
10279		break;
10280	case BPF_FUNC_redirect_map:
10281		if (map->map_type != BPF_MAP_TYPE_DEVMAP &&
10282		    map->map_type != BPF_MAP_TYPE_DEVMAP_HASH &&
10283		    map->map_type != BPF_MAP_TYPE_CPUMAP &&
10284		    map->map_type != BPF_MAP_TYPE_XSKMAP)
10285			goto error;
10286		break;
10287	case BPF_FUNC_sk_redirect_map:
10288	case BPF_FUNC_msg_redirect_map:
10289	case BPF_FUNC_sock_map_update:
10290		if (map->map_type != BPF_MAP_TYPE_SOCKMAP)
10291			goto error;
10292		break;
10293	case BPF_FUNC_sk_redirect_hash:
10294	case BPF_FUNC_msg_redirect_hash:
10295	case BPF_FUNC_sock_hash_update:
10296		if (map->map_type != BPF_MAP_TYPE_SOCKHASH)
10297			goto error;
10298		break;
10299	case BPF_FUNC_get_local_storage:
10300		if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
10301		    map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
10302			goto error;
10303		break;
10304	case BPF_FUNC_sk_select_reuseport:
10305		if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY &&
10306		    map->map_type != BPF_MAP_TYPE_SOCKMAP &&
10307		    map->map_type != BPF_MAP_TYPE_SOCKHASH)
10308			goto error;
10309		break;
10310	case BPF_FUNC_map_pop_elem:
10311		if (map->map_type != BPF_MAP_TYPE_QUEUE &&
10312		    map->map_type != BPF_MAP_TYPE_STACK)
10313			goto error;
10314		break;
10315	case BPF_FUNC_map_peek_elem:
10316	case BPF_FUNC_map_push_elem:
10317		if (map->map_type != BPF_MAP_TYPE_QUEUE &&
10318		    map->map_type != BPF_MAP_TYPE_STACK &&
10319		    map->map_type != BPF_MAP_TYPE_BLOOM_FILTER)
10320			goto error;
10321		break;
10322	case BPF_FUNC_map_lookup_percpu_elem:
10323		if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY &&
10324		    map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
10325		    map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH)
10326			goto error;
10327		break;
10328	case BPF_FUNC_sk_storage_get:
10329	case BPF_FUNC_sk_storage_delete:
10330		if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
10331			goto error;
10332		break;
10333	case BPF_FUNC_inode_storage_get:
10334	case BPF_FUNC_inode_storage_delete:
10335		if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE)
10336			goto error;
10337		break;
10338	case BPF_FUNC_task_storage_get:
10339	case BPF_FUNC_task_storage_delete:
10340		if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE)
10341			goto error;
10342		break;
10343	case BPF_FUNC_cgrp_storage_get:
10344	case BPF_FUNC_cgrp_storage_delete:
10345		if (map->map_type != BPF_MAP_TYPE_CGRP_STORAGE)
10346			goto error;
10347		break;
10348	default:
10349		break;
10350	}
10351
10352	return 0;
10353error:
10354	verbose(env, "cannot pass map_type %d into func %s#%d\n",
10355		map->map_type, func_id_name(func_id), func_id);
10356	return -EINVAL;
10357}
10358
10359static bool check_raw_mode_ok(const struct bpf_func_proto *fn)
10360{
10361	int count = 0;
10362
10363	if (arg_type_is_raw_mem(fn->arg1_type))
10364		count++;
10365	if (arg_type_is_raw_mem(fn->arg2_type))
10366		count++;
10367	if (arg_type_is_raw_mem(fn->arg3_type))
10368		count++;
10369	if (arg_type_is_raw_mem(fn->arg4_type))
10370		count++;
10371	if (arg_type_is_raw_mem(fn->arg5_type))
10372		count++;
10373
10374	/* We only support one arg being in raw mode at the moment,
10375	 * which is sufficient for the helper functions we have
10376	 * right now.
10377	 */
10378	return count <= 1;
10379}
10380
10381static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg)
10382{
10383	bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE;
10384	bool has_size = fn->arg_size[arg] != 0;
10385	bool is_next_size = false;
10386
10387	if (arg + 1 < ARRAY_SIZE(fn->arg_type))
10388		is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]);
10389
10390	if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM)
10391		return is_next_size;
10392
10393	return has_size == is_next_size || is_next_size == is_fixed;
10394}
10395
10396static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
10397{
10398	/* bpf_xxx(..., buf, len) call will access 'len'
10399	 * bytes from memory 'buf'. Both arg types need
10400	 * to be paired, so make sure there's no buggy
10401	 * helper function specification.
10402	 */
10403	if (arg_type_is_mem_size(fn->arg1_type) ||
10404	    check_args_pair_invalid(fn, 0) ||
10405	    check_args_pair_invalid(fn, 1) ||
10406	    check_args_pair_invalid(fn, 2) ||
10407	    check_args_pair_invalid(fn, 3) ||
10408	    check_args_pair_invalid(fn, 4))
10409		return false;
10410
10411	return true;
10412}
10413
10414static bool check_btf_id_ok(const struct bpf_func_proto *fn)
10415{
10416	int i;
10417
10418	for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
10419		if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID)
10420			return !!fn->arg_btf_id[i];
10421		if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK)
10422			return fn->arg_btf_id[i] == BPF_PTR_POISON;
10423		if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] &&
10424		    /* arg_btf_id and arg_size are in a union. */
10425		    (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM ||
10426		     !(fn->arg_type[i] & MEM_FIXED_SIZE)))
10427			return false;
10428	}
10429
10430	return true;
10431}
10432
10433static int check_func_proto(const struct bpf_func_proto *fn, int func_id)
10434{
10435	return check_raw_mode_ok(fn) &&
10436	       check_arg_pair_ok(fn) &&
10437	       check_btf_id_ok(fn) ? 0 : -EINVAL;
10438}
10439
10440/* Packet data might have moved, any old PTR_TO_PACKET[_META,_END]
10441 * are now invalid, so turn them into unknown SCALAR_VALUE.
10442 *
10443 * This also applies to dynptr slices belonging to skb and xdp dynptrs,
10444 * since these slices point to packet data.
10445 */
10446static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
10447{
10448	struct bpf_func_state *state;
10449	struct bpf_reg_state *reg;
10450
10451	bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
10452		if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg))
10453			mark_reg_invalid(env, reg);
10454	}));
10455}
10456
10457enum {
10458	AT_PKT_END = -1,
10459	BEYOND_PKT_END = -2,
10460};
10461
10462static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open)
10463{
10464	struct bpf_func_state *state = vstate->frame[vstate->curframe];
10465	struct bpf_reg_state *reg = &state->regs[regn];
10466
10467	if (reg->type != PTR_TO_PACKET)
10468		/* PTR_TO_PACKET_META is not supported yet */
10469		return;
10470
10471	/* The 'reg' is pkt > pkt_end or pkt >= pkt_end.
10472	 * How far beyond pkt_end it goes is unknown.
10473	 * if (!range_open) it's the case of pkt >= pkt_end
10474	 * if (range_open) it's the case of pkt > pkt_end
10475	 * hence this pointer is at least 1 byte bigger than pkt_end
10476	 */
10477	if (range_open)
10478		reg->range = BEYOND_PKT_END;
10479	else
10480		reg->range = AT_PKT_END;
10481}
10482
10483static int release_reference_nomark(struct bpf_verifier_state *state, int ref_obj_id)
10484{
10485	int i;
10486
10487	for (i = 0; i < state->acquired_refs; i++) {
10488		if (state->refs[i].type != REF_TYPE_PTR)
10489			continue;
10490		if (state->refs[i].id == ref_obj_id) {
10491			release_reference_state(state, i);
10492			return 0;
10493		}
10494	}
10495	return -EINVAL;
10496}
10497
10498/* The pointer with the specified id has released its reference to kernel
10499 * resources. Identify all copies of the same pointer and clear the reference.
10500 *
10501 * This is the release function corresponding to acquire_reference(). Idempotent.
10502 */
10503static int release_reference(struct bpf_verifier_env *env, int ref_obj_id)
10504{
10505	struct bpf_verifier_state *vstate = env->cur_state;
10506	struct bpf_func_state *state;
10507	struct bpf_reg_state *reg;
10508	int err;
10509
10510	err = release_reference_nomark(vstate, ref_obj_id);
10511	if (err)
10512		return err;
10513
10514	bpf_for_each_reg_in_vstate(vstate, state, reg, ({
10515		if (reg->ref_obj_id == ref_obj_id)
10516			mark_reg_invalid(env, reg);
10517	}));
10518
10519	return 0;
10520}
10521
10522static void invalidate_non_owning_refs(struct bpf_verifier_env *env)
10523{
10524	struct bpf_func_state *unused;
10525	struct bpf_reg_state *reg;
10526
10527	bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({
10528		if (type_is_non_owning_ref(reg->type))
10529			mark_reg_invalid(env, reg);
10530	}));
10531}
10532
10533static void clear_caller_saved_regs(struct bpf_verifier_env *env,
10534				    struct bpf_reg_state *regs)
10535{
10536	int i;
10537
10538	/* after the call registers r0 - r5 were scratched */
10539	for (i = 0; i < CALLER_SAVED_REGS; i++) {
10540		mark_reg_not_init(env, regs, caller_saved[i]);
10541		__check_reg_arg(env, regs, caller_saved[i], DST_OP_NO_MARK);
10542	}
10543}
10544
10545typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
10546				   struct bpf_func_state *caller,
10547				   struct bpf_func_state *callee,
10548				   int insn_idx);
10549
10550static int set_callee_state(struct bpf_verifier_env *env,
10551			    struct bpf_func_state *caller,
10552			    struct bpf_func_state *callee, int insn_idx);
10553
10554static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite,
10555			    set_callee_state_fn set_callee_state_cb,
10556			    struct bpf_verifier_state *state)
10557{
10558	struct bpf_func_state *caller, *callee;
10559	int err;
10560
10561	if (state->curframe + 1 >= MAX_CALL_FRAMES) {
10562		verbose(env, "the call stack of %d frames is too deep\n",
10563			state->curframe + 2);
10564		return -E2BIG;
10565	}
10566
10567	if (state->frame[state->curframe + 1]) {
10568		verifier_bug(env, "Frame %d already allocated", state->curframe + 1);
10569		return -EFAULT;
10570	}
10571
10572	caller = state->frame[state->curframe];
10573	callee = kzalloc(sizeof(*callee), GFP_KERNEL_ACCOUNT);
10574	if (!callee)
10575		return -ENOMEM;
10576	state->frame[state->curframe + 1] = callee;
10577
10578	/* callee cannot access r0, r6 - r9 for reading and has to write
10579	 * into its own stack before reading from it.
10580	 * callee can read/write into caller's stack
10581	 */
10582	init_func_state(env, callee,
10583			/* remember the callsite, it will be used by bpf_exit */
10584			callsite,
10585			state->curframe + 1 /* frameno within this callchain */,
10586			subprog /* subprog number within this prog */);
10587	err = set_callee_state_cb(env, caller, callee, callsite);
10588	if (err)
10589		goto err_out;
10590
10591	/* only increment it after check_reg_arg() finished */
10592	state->curframe++;
10593
10594	return 0;
10595
10596err_out:
10597	free_func_state(callee);
10598	state->frame[state->curframe + 1] = NULL;
10599	return err;
10600}
10601
10602static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
10603				    const struct btf *btf,
10604				    struct bpf_reg_state *regs)
10605{
10606	struct bpf_subprog_info *sub = subprog_info(env, subprog);
10607	struct bpf_verifier_log *log = &env->log;
10608	u32 i;
10609	int ret;
10610
10611	ret = btf_prepare_func_args(env, subprog);
10612	if (ret)
10613		return ret;
10614
10615	/* check that BTF function arguments match actual types that the
10616	 * verifier sees.
10617	 */
10618	for (i = 0; i < sub->arg_cnt; i++) {
10619		u32 regno = i + 1;
10620		struct bpf_reg_state *reg = &regs[regno];
10621		struct bpf_subprog_arg_info *arg = &sub->args[i];
10622
10623		if (arg->arg_type == ARG_ANYTHING) {
10624			if (reg->type != SCALAR_VALUE) {
10625				bpf_log(log, "R%d is not a scalar\n", regno);
10626				return -EINVAL;
10627			}
10628		} else if (arg->arg_type & PTR_UNTRUSTED) {
10629			/*
10630			 * Anything is allowed for untrusted arguments, as these are
10631			 * read-only and probe read instructions would protect against
10632			 * invalid memory access.
10633			 */
10634		} else if (arg->arg_type == ARG_PTR_TO_CTX) {
10635			ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE);
10636			if (ret < 0)
10637				return ret;
10638			/* If function expects ctx type in BTF check that caller
10639			 * is passing PTR_TO_CTX.
10640			 */
10641			if (reg->type != PTR_TO_CTX) {
10642				bpf_log(log, "arg#%d expects pointer to ctx\n", i);
10643				return -EINVAL;
10644			}
10645		} else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) {
10646			ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE);
10647			if (ret < 0)
10648				return ret;
10649			if (check_mem_reg(env, reg, regno, arg->mem_size))
10650				return -EINVAL;
10651			if (!(arg->arg_type & PTR_MAYBE_NULL) && (reg->type & PTR_MAYBE_NULL)) {
10652				bpf_log(log, "arg#%d is expected to be non-NULL\n", i);
10653				return -EINVAL;
10654			}
10655		} else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) {
10656			/*
10657			 * Can pass any value and the kernel won't crash, but
10658			 * only PTR_TO_ARENA or SCALAR make sense. Everything
10659			 * else is a bug in the bpf program. Point it out to
10660			 * the user at the verification time instead of
10661			 * run-time debug nightmare.
10662			 */
10663			if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) {
10664				bpf_log(log, "R%d is not a pointer to arena or scalar.\n", regno);
10665				return -EINVAL;
10666			}
10667		} else if (arg->arg_type == (ARG_PTR_TO_DYNPTR | MEM_RDONLY)) {
10668			ret = check_func_arg_reg_off(env, reg, regno, ARG_PTR_TO_DYNPTR);
10669			if (ret)
10670				return ret;
10671
10672			ret = process_dynptr_func(env, regno, -1, arg->arg_type, 0);
10673			if (ret)
10674				return ret;
10675		} else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) {
10676			struct bpf_call_arg_meta meta;
10677			int err;
10678
10679			if (register_is_null(reg) && type_may_be_null(arg->arg_type))
10680				continue;
10681
10682			memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */
10683			err = check_reg_type(env, regno, arg->arg_type, &arg->btf_id, &meta);
10684			err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type);
10685			if (err)
10686				return err;
10687		} else {
10688			verifier_bug(env, "unrecognized arg#%d type %d", i, arg->arg_type);
10689			return -EFAULT;
10690		}
10691	}
10692
10693	return 0;
10694}
10695
10696/* Compare BTF of a function call with given bpf_reg_state.
10697 * Returns:
10698 * EFAULT - there is a verifier bug. Abort verification.
10699 * EINVAL - there is a type mismatch or BTF is not available.
10700 * 0 - BTF matches with what bpf_reg_state expects.
10701 * Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
10702 */
10703static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,
10704				  struct bpf_reg_state *regs)
10705{
10706	struct bpf_prog *prog = env->prog;
10707	struct btf *btf = prog->aux->btf;
10708	u32 btf_id;
10709	int err;
10710
10711	if (!prog->aux->func_info)
10712		return -EINVAL;
10713
10714	btf_id = prog->aux->func_info[subprog].type_id;
10715	if (!btf_id)
10716		return -EFAULT;
10717
10718	if (prog->aux->func_info_aux[subprog].unreliable)
10719		return -EINVAL;
10720
10721	err = btf_check_func_arg_match(env, subprog, btf, regs);
10722	/* Compiler optimizations can remove arguments from static functions
10723	 * or mismatched type can be passed into a global function.
10724	 * In such cases mark the function as unreliable from BTF point of view.
10725	 */
10726	if (err)
10727		prog->aux->func_info_aux[subprog].unreliable = true;
10728	return err;
10729}
10730
10731static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
10732			      int insn_idx, int subprog,
10733			      set_callee_state_fn set_callee_state_cb)
10734{
10735	struct bpf_verifier_state *state = env->cur_state, *callback_state;
10736	struct bpf_func_state *caller, *callee;
10737	int err;
10738
10739	caller = state->frame[state->curframe];
10740	err = btf_check_subprog_call(env, subprog, caller->regs);
10741	if (err == -EFAULT)
10742		return err;
10743
10744	/* set_callee_state is used for direct subprog calls, but we are
10745	 * interested in validating only BPF helpers that can call subprogs as
10746	 * callbacks
10747	 */
10748	env->subprog_info[subprog].is_cb = true;
10749	if (bpf_pseudo_kfunc_call(insn) &&
10750	    !is_callback_calling_kfunc(insn->imm)) {
10751		verifier_bug(env, "kfunc %s#%d not marked as callback-calling",
10752			     func_id_name(insn->imm), insn->imm);
10753		return -EFAULT;
10754	} else if (!bpf_pseudo_kfunc_call(insn) &&
10755		   !is_callback_calling_function(insn->imm)) { /* helper */
10756		verifier_bug(env, "helper %s#%d not marked as callback-calling",
10757			     func_id_name(insn->imm), insn->imm);
10758		return -EFAULT;
10759	}
10760
10761	if (is_async_callback_calling_insn(insn)) {
10762		struct bpf_verifier_state *async_cb;
10763
10764		/* there is no real recursion here. timer and workqueue callbacks are async */
10765		env->subprog_info[subprog].is_async_cb = true;
10766		async_cb = push_async_cb(env, env->subprog_info[subprog].start,
10767					 insn_idx, subprog,
10768					 is_async_cb_sleepable(env, insn));
10769		if (IS_ERR(async_cb))
10770			return PTR_ERR(async_cb);
10771		callee = async_cb->frame[0];
10772		callee->async_entry_cnt = caller->async_entry_cnt + 1;
10773
10774		/* Convert bpf_timer_set_callback() args into timer callback args */
10775		err = set_callee_state_cb(env, caller, callee, insn_idx);
10776		if (err)
10777			return err;
10778
10779		return 0;
10780	}
10781
10782	/* for callback functions enqueue entry to callback and
10783	 * proceed with next instruction within current frame.
10784	 */
10785	callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false);
10786	if (IS_ERR(callback_state))
10787		return PTR_ERR(callback_state);
10788
10789	err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb,
10790			       callback_state);
10791	if (err)
10792		return err;
10793
10794	callback_state->callback_unroll_depth++;
10795	callback_state->frame[callback_state->curframe - 1]->callback_depth++;
10796	caller->callback_depth = 0;
10797	return 0;
10798}
10799
10800static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
10801			   int *insn_idx)
10802{
10803	struct bpf_verifier_state *state = env->cur_state;
10804	struct bpf_func_state *caller;
10805	int err, subprog, target_insn;
10806
10807	target_insn = *insn_idx + insn->imm + 1;
10808	subprog = find_subprog(env, target_insn);
10809	if (verifier_bug_if(subprog < 0, env, "target of func call at insn %d is not a program",
10810			    target_insn))
10811		return -EFAULT;
10812
10813	caller = state->frame[state->curframe];
10814	err = btf_check_subprog_call(env, subprog, caller->regs);
10815	if (err == -EFAULT)
10816		return err;
10817	if (subprog_is_global(env, subprog)) {
10818		const char *sub_name = subprog_name(env, subprog);
10819
10820		if (env->cur_state->active_locks) {
10821			verbose(env, "global function calls are not allowed while holding a lock,\n"
10822				     "use static function instead\n");
10823			return -EINVAL;
10824		}
10825
10826		if (env->subprog_info[subprog].might_sleep &&
10827		    (env->cur_state->active_rcu_locks || env->cur_state->active_preempt_locks ||
10828		     env->cur_state->active_irq_id || !in_sleepable(env))) {
10829			verbose(env, "global functions that may sleep are not allowed in non-sleepable context,\n"
10830				     "i.e., in a RCU/IRQ/preempt-disabled section, or in\n"
10831				     "a non-sleepable BPF program context\n");
10832			return -EINVAL;
10833		}
10834
10835		if (err) {
10836			verbose(env, "Caller passes invalid args into func#%d ('%s')\n",
10837				subprog, sub_name);
10838			return err;
10839		}
10840
10841		if (env->log.level & BPF_LOG_LEVEL)
10842			verbose(env, "Func#%d ('%s') is global and assumed valid.\n",
10843				subprog, sub_name);
10844		if (env->subprog_info[subprog].changes_pkt_data)
10845			clear_all_pkt_pointers(env);
10846		/* mark global subprog for verifying after main prog */
10847		subprog_aux(env, subprog)->called = true;
10848		clear_caller_saved_regs(env, caller->regs);
10849
10850		/* All global functions return a 64-bit SCALAR_VALUE */
10851		mark_reg_unknown(env, caller->regs, BPF_REG_0);
10852		caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
10853
10854		/* continue with next insn after call */
10855		return 0;
10856	}
10857
10858	/* for regular function entry setup new frame and continue
10859	 * from that frame.
10860	 */
10861	err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state);
10862	if (err)
10863		return err;
10864
10865	clear_caller_saved_regs(env, caller->regs);
10866
10867	/* and go analyze first insn of the callee */
10868	*insn_idx = env->subprog_info[subprog].start - 1;
10869
10870	bpf_reset_live_stack_callchain(env);
10871
10872	if (env->log.level & BPF_LOG_LEVEL) {
10873		verbose(env, "caller:\n");
10874		print_verifier_state(env, state, caller->frameno, true);
10875		verbose(env, "callee:\n");
10876		print_verifier_state(env, state, state->curframe, true);
10877	}
10878
10879	return 0;
10880}
10881
10882int map_set_for_each_callback_args(struct bpf_verifier_env *env,
10883				   struct bpf_func_state *caller,
10884				   struct bpf_func_state *callee)
10885{
10886	/* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn,
10887	 *      void *callback_ctx, u64 flags);
10888	 * callback_fn(struct bpf_map *map, void *key, void *value,
10889	 *      void *callback_ctx);
10890	 */
10891	callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1];
10892
10893	callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
10894	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
10895	callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr;
10896
10897	callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
10898	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
10899	callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr;
10900
10901	/* pointer to stack or null */
10902	callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3];
10903
10904	/* unused */
10905	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
10906	return 0;
10907}
10908
10909static int set_callee_state(struct bpf_verifier_env *env,
10910			    struct bpf_func_state *caller,
10911			    struct bpf_func_state *callee, int insn_idx)
10912{
10913	int i;
10914
10915	/* copy r1 - r5 args that callee can access.  The copy includes parent
10916	 * pointers, which connects us up to the liveness chain
10917	 */
10918	for (i = BPF_REG_1; i <= BPF_REG_5; i++)
10919		callee->regs[i] = caller->regs[i];
10920	return 0;
10921}
10922
10923static int set_map_elem_callback_state(struct bpf_verifier_env *env,
10924				       struct bpf_func_state *caller,
10925				       struct bpf_func_state *callee,
10926				       int insn_idx)
10927{
10928	struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx];
10929	struct bpf_map *map;
10930	int err;
10931
10932	/* valid map_ptr and poison value does not matter */
10933	map = insn_aux->map_ptr_state.map_ptr;
10934	if (!map->ops->map_set_for_each_callback_args ||
10935	    !map->ops->map_for_each_callback) {
10936		verbose(env, "callback function not allowed for map\n");
10937		return -ENOTSUPP;
10938	}
10939
10940	err = map->ops->map_set_for_each_callback_args(env, caller, callee);
10941	if (err)
10942		return err;
10943
10944	callee->in_callback_fn = true;
10945	callee->callback_ret_range = retval_range(0, 1);
10946	return 0;
10947}
10948
10949static int set_loop_callback_state(struct bpf_verifier_env *env,
10950				   struct bpf_func_state *caller,
10951				   struct bpf_func_state *callee,
10952				   int insn_idx)
10953{
10954	/* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx,
10955	 *	    u64 flags);
10956	 * callback_fn(u64 index, void *callback_ctx);
10957	 */
10958	callee->regs[BPF_REG_1].type = SCALAR_VALUE;
10959	callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];
10960
10961	/* unused */
10962	__mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
10963	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
10964	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
10965
10966	callee->in_callback_fn = true;
10967	callee->callback_ret_range = retval_range(0, 1);
10968	return 0;
10969}
10970
10971static int set_timer_callback_state(struct bpf_verifier_env *env,
10972				    struct bpf_func_state *caller,
10973				    struct bpf_func_state *callee,
10974				    int insn_idx)
10975{
10976	struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr;
10977
10978	/* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn);
10979	 * callback_fn(struct bpf_map *map, void *key, void *value);
10980	 */
10981	callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP;
10982	__mark_reg_known_zero(&callee->regs[BPF_REG_1]);
10983	callee->regs[BPF_REG_1].map_ptr = map_ptr;
10984
10985	callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
10986	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
10987	callee->regs[BPF_REG_2].map_ptr = map_ptr;
10988
10989	callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
10990	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
10991	callee->regs[BPF_REG_3].map_ptr = map_ptr;
10992
10993	/* unused */
10994	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
10995	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
10996	callee->in_async_callback_fn = true;
10997	callee->callback_ret_range = retval_range(0, 0);
10998	return 0;
10999}
11000
11001static int set_find_vma_callback_state(struct bpf_verifier_env *env,
11002				       struct bpf_func_state *caller,
11003				       struct bpf_func_state *callee,
11004				       int insn_idx)
11005{
11006	/* bpf_find_vma(struct task_struct *task, u64 addr,
11007	 *               void *callback_fn, void *callback_ctx, u64 flags)
11008	 * (callback_fn)(struct task_struct *task,
11009	 *               struct vm_area_struct *vma, void *callback_ctx);
11010	 */
11011	callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1];
11012
11013	callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID;
11014	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
11015	callee->regs[BPF_REG_2].btf =  btf_vmlinux;
11016	callee->regs[BPF_REG_2].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA];
11017
11018	/* pointer to stack or null */
11019	callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4];
11020
11021	/* unused */
11022	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
11023	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
11024	callee->in_callback_fn = true;
11025	callee->callback_ret_range = retval_range(0, 1);
11026	return 0;
11027}
11028
11029static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env,
11030					   struct bpf_func_state *caller,
11031					   struct bpf_func_state *callee,
11032					   int insn_idx)
11033{
11034	/* bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void
11035	 *			  callback_ctx, u64 flags);
11036	 * callback_fn(const struct bpf_dynptr_t* dynptr, void *callback_ctx);
11037	 */
11038	__mark_reg_not_init(env, &callee->regs[BPF_REG_0]);
11039	mark_dynptr_cb_reg(env, &callee->regs[BPF_REG_1], BPF_DYNPTR_TYPE_LOCAL);
11040	callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];
11041
11042	/* unused */
11043	__mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
11044	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
11045	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
11046
11047	callee->in_callback_fn = true;
11048	callee->callback_ret_range = retval_range(0, 1);
11049	return 0;
11050}
11051
11052static int set_rbtree_add_callback_state(struct bpf_verifier_env *env,
11053					 struct bpf_func_state *caller,
11054					 struct bpf_func_state *callee,
11055					 int insn_idx)
11056{
11057	/* void bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node,
11058	 *                     bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b));
11059	 *
11060	 * 'struct bpf_rb_node *node' arg to bpf_rbtree_add_impl is the same PTR_TO_BTF_ID w/ offset
11061	 * that 'less' callback args will be receiving. However, 'node' arg was release_reference'd
11062	 * by this point, so look at 'root'
11063	 */
11064	struct btf_field *field;
11065
11066	field = reg_find_field_offset(&caller->regs[BPF_REG_1], caller->regs[BPF_REG_1].off,
11067				      BPF_RB_ROOT);
11068	if (!field || !field->graph_root.value_btf_id)
11069		return -EFAULT;
11070
11071	mark_reg_graph_node(callee->regs, BPF_REG_1, &field->graph_root);
11072	ref_set_non_owning(env, &callee->regs[BPF_REG_1]);
11073	mark_reg_graph_node(callee->regs, BPF_REG_2, &field->graph_root);
11074	ref_set_non_owning(env, &callee->regs[BPF_REG_2]);
11075
11076	__mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
11077	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
11078	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
11079	callee->in_callback_fn = true;
11080	callee->callback_ret_range = retval_range(0, 1);
11081	return 0;
11082}
11083
11084static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env,
11085						 struct bpf_func_state *caller,
11086						 struct bpf_func_state *callee,
11087						 int insn_idx)
11088{
11089	struct bpf_map *map_ptr = caller->regs[BPF_REG_3].map_ptr;
11090
11091	/*
11092	 * callback_fn(struct bpf_map *map, void *key, void *value);
11093	 */
11094	callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP;
11095	__mark_reg_known_zero(&callee->regs[BPF_REG_1]);
11096	callee->regs[BPF_REG_1].map_ptr = map_ptr;
11097
11098	callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
11099	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
11100	callee->regs[BPF_REG_2].map_ptr = map_ptr;
11101
11102	callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
11103	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
11104	callee->regs[BPF_REG_3].map_ptr = map_ptr;
11105
11106	/* unused */
11107	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
11108	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
11109	callee->in_async_callback_fn = true;
11110	callee->callback_ret_range = retval_range(S32_MIN, S32_MAX);
11111	return 0;
11112}
11113
11114static bool is_rbtree_lock_required_kfunc(u32 btf_id);
11115
11116/* Are we currently verifying the callback for a rbtree helper that must
11117 * be called with lock held? If so, no need to complain about unreleased
11118 * lock
11119 */
11120static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env)
11121{
11122	struct bpf_verifier_state *state = env->cur_state;
11123	struct bpf_insn *insn = env->prog->insnsi;
11124	struct bpf_func_state *callee;
11125	int kfunc_btf_id;
11126
11127	if (!state->curframe)
11128		return false;
11129
11130	callee = state->frame[state->curframe];
11131
11132	if (!callee->in_callback_fn)
11133		return false;
11134
11135	kfunc_btf_id = insn[callee->callsite].imm;
11136	return is_rbtree_lock_required_kfunc(kfunc_btf_id);
11137}
11138
11139static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg,
11140				bool return_32bit)
11141{
11142	if (return_32bit)
11143		return range.minval <= reg->s32_min_value && reg->s32_max_value <= range.maxval;
11144	else
11145		return range.minval <= reg->smin_value && reg->smax_value <= range.maxval;
11146}
11147
11148static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
11149{
11150	struct bpf_verifier_state *state = env->cur_state, *prev_st;
11151	struct bpf_func_state *caller, *callee;
11152	struct bpf_reg_state *r0;
11153	bool in_callback_fn;
11154	int err;
11155
11156	err = bpf_update_live_stack(env);
11157	if (err)
11158		return err;
11159
11160	callee = state->frame[state->curframe];
11161	r0 = &callee->regs[BPF_REG_0];
11162	if (r0->type == PTR_TO_STACK) {
11163		/* technically it's ok to return caller's stack pointer
11164		 * (or caller's caller's pointer) back to the caller,
11165		 * since these pointers are valid. Only current stack
11166		 * pointer will be invalid as soon as function exits,
11167		 * but let's be conservative
11168		 */
11169		verbose(env, "cannot return stack pointer to the caller\n");
11170		return -EINVAL;
11171	}
11172
11173	caller = state->frame[state->curframe - 1];
11174	if (callee->in_callback_fn) {
11175		if (r0->type != SCALAR_VALUE) {
11176			verbose(env, "R0 not a scalar value\n");
11177			return -EACCES;
11178		}
11179
11180		/* we are going to rely on register's precise value */
11181		err = mark_chain_precision(env, BPF_REG_0);
11182		if (err)
11183			return err;
11184
11185		/* enforce R0 return value range, and bpf_callback_t returns 64bit */
11186		if (!retval_range_within(callee->callback_ret_range, r0, false)) {
11187			verbose_invalid_scalar(env, r0, callee->callback_ret_range,
11188					       "At callback return", "R0");
11189			return -EINVAL;
11190		}
11191		if (!bpf_calls_callback(env, callee->callsite)) {
11192			verifier_bug(env, "in callback at %d, callsite %d !calls_callback",
11193				     *insn_idx, callee->callsite);
11194			return -EFAULT;
11195		}
11196	} else {
11197		/* return to the caller whatever r0 had in the callee */
11198		caller->regs[BPF_REG_0] = *r0;
11199	}
11200
11201	/* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite,
11202	 * there function call logic would reschedule callback visit. If iteration
11203	 * converges is_state_visited() would prune that visit eventually.
11204	 */
11205	in_callback_fn = callee->in_callback_fn;
11206	if (in_callback_fn)
11207		*insn_idx = callee->callsite;
11208	else
11209		*insn_idx = callee->callsite + 1;
11210
11211	if (env->log.level & BPF_LOG_LEVEL) {
11212		verbose(env, "returning from callee:\n");
11213		print_verifier_state(env, state, callee->frameno, true);
11214		verbose(env, "to caller at %d:\n", *insn_idx);
11215		print_verifier_state(env, state, caller->frameno, true);
11216	}
11217	/* clear everything in the callee. In case of exceptional exits using
11218	 * bpf_throw, this will be done by copy_verifier_state for extra frames. */
11219	free_func_state(callee);
11220	state->frame[state->curframe--] = NULL;
11221
11222	/* for callbacks widen imprecise scalars to make programs like below verify:
11223	 *
11224	 *   struct ctx { int i; }
11225	 *   void cb(int idx, struct ctx *ctx) { ctx->i++; ... }
11226	 *   ...
11227	 *   struct ctx = { .i = 0; }
11228	 *   bpf_loop(100, cb, &ctx, 0);
11229	 *
11230	 * This is similar to what is done in process_iter_next_call() for open
11231	 * coded iterators.
11232	 */
11233	prev_st = in_callback_fn ? find_prev_entry(env, state, *insn_idx) : NULL;
11234	if (prev_st) {
11235		err = widen_imprecise_scalars(env, prev_st, state);
11236		if (err)
11237			return err;
11238	}
11239	return 0;
11240}
11241
11242static int do_refine_retval_range(struct bpf_verifier_env *env,
11243				  struct bpf_reg_state *regs, int ret_type,
11244				  int func_id,
11245				  struct bpf_call_arg_meta *meta)
11246{
11247	struct bpf_reg_state *ret_reg = &regs[BPF_REG_0];
11248
11249	if (ret_type != RET_INTEGER)
11250		return 0;
11251
11252	switch (func_id) {
11253	case BPF_FUNC_get_stack:
11254	case BPF_FUNC_get_task_stack:
11255	case BPF_FUNC_probe_read_str:
11256	case BPF_FUNC_probe_read_kernel_str:
11257	case BPF_FUNC_probe_read_user_str:
11258		ret_reg->smax_value = meta->msize_max_value;
11259		ret_reg->s32_max_value = meta->msize_max_value;
11260		ret_reg->smin_value = -MAX_ERRNO;
11261		ret_reg->s32_min_value = -MAX_ERRNO;
11262		reg_bounds_sync(ret_reg);
11263		break;
11264	case BPF_FUNC_get_smp_processor_id:
11265		ret_reg->umax_value = nr_cpu_ids - 1;
11266		ret_reg->u32_max_value = nr_cpu_ids - 1;
11267		ret_reg->smax_value = nr_cpu_ids - 1;
11268		ret_reg->s32_max_value = nr_cpu_ids - 1;
11269		ret_reg->umin_value = 0;
11270		ret_reg->u32_min_value = 0;
11271		ret_reg->smin_value = 0;
11272		ret_reg->s32_min_value = 0;
11273		reg_bounds_sync(ret_reg);
11274		break;
11275	}
11276
11277	return reg_bounds_sanity_check(env, ret_reg, "retval");
11278}
11279
11280static int
11281record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
11282		int func_id, int insn_idx)
11283{
11284	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
11285	struct bpf_map *map = meta->map_ptr;
11286
11287	if (func_id != BPF_FUNC_tail_call &&
11288	    func_id != BPF_FUNC_map_lookup_elem &&
11289	    func_id != BPF_FUNC_map_update_elem &&
11290	    func_id != BPF_FUNC_map_delete_elem &&
11291	    func_id != BPF_FUNC_map_push_elem &&
11292	    func_id != BPF_FUNC_map_pop_elem &&
11293	    func_id != BPF_FUNC_map_peek_elem &&
11294	    func_id != BPF_FUNC_for_each_map_elem &&
11295	    func_id != BPF_FUNC_redirect_map &&
11296	    func_id != BPF_FUNC_map_lookup_percpu_elem)
11297		return 0;
11298
11299	if (map == NULL) {
11300		verifier_bug(env, "expected map for helper call");
11301		return -EFAULT;
11302	}
11303
11304	/* In case of read-only, some additional restrictions
11305	 * need to be applied in order to prevent altering the
11306	 * state of the map from program side.
11307	 */
11308	if ((map->map_flags & BPF_F_RDONLY_PROG) &&
11309	    (func_id == BPF_FUNC_map_delete_elem ||
11310	     func_id == BPF_FUNC_map_update_elem ||
11311	     func_id == BPF_FUNC_map_push_elem ||
11312	     func_id == BPF_FUNC_map_pop_elem)) {
11313		verbose(env, "write into map forbidden\n");
11314		return -EACCES;
11315	}
11316
11317	if (!aux->map_ptr_state.map_ptr)
11318		bpf_map_ptr_store(aux, meta->map_ptr,
11319				  !meta->map_ptr->bypass_spec_v1, false);
11320	else if (aux->map_ptr_state.map_ptr != meta->map_ptr)
11321		bpf_map_ptr_store(aux, meta->map_ptr,
11322				  !meta->map_ptr->bypass_spec_v1, true);
11323	return 0;
11324}
11325
11326static int
11327record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
11328		int func_id, int insn_idx)
11329{
11330	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
11331	struct bpf_reg_state *regs = cur_regs(env), *reg;
11332	struct bpf_map *map = meta->map_ptr;
11333	u64 val, max;
11334	int err;
11335
11336	if (func_id != BPF_FUNC_tail_call)
11337		return 0;
11338	if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) {
11339		verbose(env, "expected prog array map for tail call");
11340		return -EINVAL;
11341	}
11342
11343	reg = &regs[BPF_REG_3];
11344	val = reg->var_off.value;
11345	max = map->max_entries;
11346
11347	if (!(is_reg_const(reg, false) && val < max)) {
11348		bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
11349		return 0;
11350	}
11351
11352	err = mark_chain_precision(env, BPF_REG_3);
11353	if (err)
11354		return err;
11355	if (bpf_map_key_unseen(aux))
11356		bpf_map_key_store(aux, val);
11357	else if (!bpf_map_key_poisoned(aux) &&
11358		  bpf_map_key_immediate(aux) != val)
11359		bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
11360	return 0;
11361}
11362
11363static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit)
11364{
11365	struct bpf_verifier_state *state = env->cur_state;
11366	enum bpf_prog_type type = resolve_prog_type(env->prog);
11367	struct bpf_reg_state *reg = reg_state(env, BPF_REG_0);
11368	bool refs_lingering = false;
11369	int i;
11370
11371	if (!exception_exit && cur_func(env)->frameno)
11372		return 0;
11373
11374	for (i = 0; i < state->acquired_refs; i++) {
11375		if (state->refs[i].type != REF_TYPE_PTR)
11376			continue;
11377		/* Allow struct_ops programs to return a referenced kptr back to
11378		 * kernel. Type checks are performed later in check_return_code.
11379		 */
11380		if (type == BPF_PROG_TYPE_STRUCT_OPS && !exception_exit &&
11381		    reg->ref_obj_id == state->refs[i].id)
11382			continue;
11383		verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
11384			state->refs[i].id, state->refs[i].insn_idx);
11385		refs_lingering = true;
11386	}
11387	return refs_lingering ? -EINVAL : 0;
11388}
11389
11390static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit, bool check_lock, const char *prefix)
11391{
11392	int err;
11393
11394	if (check_lock && env->cur_state->active_locks) {
11395		verbose(env, "%s cannot be used inside bpf_spin_lock-ed region\n", prefix);
11396		return -EINVAL;
11397	}
11398
11399	err = check_reference_leak(env, exception_exit);
11400	if (err) {
11401		verbose(env, "%s would lead to reference leak\n", prefix);
11402		return err;
11403	}
11404
11405	if (check_lock && env->cur_state->active_irq_id) {
11406		verbose(env, "%s cannot be used inside bpf_local_irq_save-ed region\n", prefix);
11407		return -EINVAL;
11408	}
11409
11410	if (check_lock && env->cur_state->active_rcu_locks) {
11411		verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix);
11412		return -EINVAL;
11413	}
11414
11415	if (check_lock && env->cur_state->active_preempt_locks) {
11416		verbose(env, "%s cannot be used inside bpf_preempt_disable-ed region\n", prefix);
11417		return -EINVAL;
11418	}
11419
11420	return 0;
11421}
11422
11423static int check_bpf_snprintf_call(struct bpf_verifier_env *env,
11424				   struct bpf_reg_state *regs)
11425{
11426	struct bpf_reg_state *fmt_reg = &regs[BPF_REG_3];
11427	struct bpf_reg_state *data_len_reg = &regs[BPF_REG_5];
11428	struct bpf_map *fmt_map = fmt_reg->map_ptr;
11429	struct bpf_bprintf_data data = {};
11430	int err, fmt_map_off, num_args;
11431	u64 fmt_addr;
11432	char *fmt;
11433
11434	/* data must be an array of u64 */
11435	if (data_len_reg->var_off.value % 8)
11436		return -EINVAL;
11437	num_args = data_len_reg->var_off.value / 8;
11438
11439	/* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const
11440	 * and map_direct_value_addr is set.
11441	 */
11442	fmt_map_off = fmt_reg->off + fmt_reg->var_off.value;
11443	err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr,
11444						  fmt_map_off);
11445	if (err) {
11446		verbose(env, "failed to retrieve map value address\n");
11447		return -EFAULT;
11448	}
11449	fmt = (char *)(long)fmt_addr + fmt_map_off;
11450
11451	/* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we
11452	 * can focus on validating the format specifiers.
11453	 */
11454	err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, num_args, &data);
11455	if (err < 0)
11456		verbose(env, "Invalid format string\n");
11457
11458	return err;
11459}
11460
11461static int check_get_func_ip(struct bpf_verifier_env *env)
11462{
11463	enum bpf_prog_type type = resolve_prog_type(env->prog);
11464	int func_id = BPF_FUNC_get_func_ip;
11465
11466	if (type == BPF_PROG_TYPE_TRACING) {
11467		if (!bpf_prog_has_trampoline(env->prog)) {
11468			verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n",
11469				func_id_name(func_id), func_id);
11470			return -ENOTSUPP;
11471		}
11472		return 0;
11473	} else if (type == BPF_PROG_TYPE_KPROBE) {
11474		return 0;
11475	}
11476
11477	verbose(env, "func %s#%d not supported for program type %d\n",
11478		func_id_name(func_id), func_id, type);
11479	return -ENOTSUPP;
11480}
11481
11482static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env)
11483{
11484	return &env->insn_aux_data[env->insn_idx];
11485}
11486
11487static bool loop_flag_is_zero(struct bpf_verifier_env *env)
11488{
11489	struct bpf_reg_state *regs = cur_regs(env);
11490	struct bpf_reg_state *reg = &regs[BPF_REG_4];
11491	bool reg_is_null = register_is_null(reg);
11492
11493	if (reg_is_null)
11494		mark_chain_precision(env, BPF_REG_4);
11495
11496	return reg_is_null;
11497}
11498
11499static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno)
11500{
11501	struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state;
11502
11503	if (!state->initialized) {
11504		state->initialized = 1;
11505		state->fit_for_inline = loop_flag_is_zero(env);
11506		state->callback_subprogno = subprogno;
11507		return;
11508	}
11509
11510	if (!state->fit_for_inline)
11511		return;
11512
11513	state->fit_for_inline = (loop_flag_is_zero(env) &&
11514				 state->callback_subprogno == subprogno);
11515}
11516
11517/* Returns whether or not the given map type can potentially elide
11518 * lookup return value nullness check. This is possible if the key
11519 * is statically known.
11520 */
11521static bool can_elide_value_nullness(enum bpf_map_type type)
11522{
11523	switch (type) {
11524	case BPF_MAP_TYPE_ARRAY:
11525	case BPF_MAP_TYPE_PERCPU_ARRAY:
11526		return true;
11527	default:
11528		return false;
11529	}
11530}
11531
11532static int get_helper_proto(struct bpf_verifier_env *env, int func_id,
11533			    const struct bpf_func_proto **ptr)
11534{
11535	if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID)
11536		return -ERANGE;
11537
11538	if (!env->ops->get_func_proto)
11539		return -EINVAL;
11540
11541	*ptr = env->ops->get_func_proto(func_id, env->prog);
11542	return *ptr && (*ptr)->func ? 0 : -EINVAL;
11543}
11544
11545/* Check if we're in a sleepable context. */
11546static inline bool in_sleepable_context(struct bpf_verifier_env *env)
11547{
11548	return !env->cur_state->active_rcu_locks &&
11549	       !env->cur_state->active_preempt_locks &&
11550	       !env->cur_state->active_irq_id &&
11551	       in_sleepable(env);
11552}
11553
11554static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
11555			     int *insn_idx_p)
11556{
11557	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
11558	bool returns_cpu_specific_alloc_ptr = false;
11559	const struct bpf_func_proto *fn = NULL;
11560	enum bpf_return_type ret_type;
11561	enum bpf_type_flag ret_flag;
11562	struct bpf_reg_state *regs;
11563	struct bpf_call_arg_meta meta;
11564	int insn_idx = *insn_idx_p;
11565	bool changes_data;
11566	int i, err, func_id;
11567
11568	/* find function prototype */
11569	func_id = insn->imm;
11570	err = get_helper_proto(env, insn->imm, &fn);
11571	if (err == -ERANGE) {
11572		verbose(env, "invalid func %s#%d\n", func_id_name(func_id), func_id);
11573		return -EINVAL;
11574	}
11575
11576	if (err) {
11577		verbose(env, "program of this type cannot use helper %s#%d\n",
11578			func_id_name(func_id), func_id);
11579		return err;
11580	}
11581
11582	/* eBPF programs must be GPL compatible to use GPL-ed functions */
11583	if (!env->prog->gpl_compatible && fn->gpl_only) {
11584		verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");
11585		return -EINVAL;
11586	}
11587
11588	if (fn->allowed && !fn->allowed(env->prog)) {
11589		verbose(env, "helper call is not allowed in probe\n");
11590		return -EINVAL;
11591	}
11592
11593	if (!in_sleepable(env) && fn->might_sleep) {
11594		verbose(env, "helper call might sleep in a non-sleepable prog\n");
11595		return -EINVAL;
11596	}
11597
11598	/* With LD_ABS/IND some JITs save/restore skb from r1. */
11599	changes_data = bpf_helper_changes_pkt_data(func_id);
11600	if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {
11601		verifier_bug(env, "func %s#%d: r1 != ctx", func_id_name(func_id), func_id);
11602		return -EFAULT;
11603	}
11604
11605	memset(&meta, 0, sizeof(meta));
11606	meta.pkt_access = fn->pkt_access;
11607
11608	err = check_func_proto(fn, func_id);
11609	if (err) {
11610		verifier_bug(env, "incorrect func proto %s#%d", func_id_name(func_id), func_id);
11611		return err;
11612	}
11613
11614	if (env->cur_state->active_rcu_locks) {
11615		if (fn->might_sleep) {
11616			verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n",
11617				func_id_name(func_id), func_id);
11618			return -EINVAL;
11619		}
11620	}
11621
11622	if (env->cur_state->active_preempt_locks) {
11623		if (fn->might_sleep) {
11624			verbose(env, "sleepable helper %s#%d in non-preemptible region\n",
11625				func_id_name(func_id), func_id);
11626			return -EINVAL;
11627		}
11628	}
11629
11630	if (env->cur_state->active_irq_id) {
11631		if (fn->might_sleep) {
11632			verbose(env, "sleepable helper %s#%d in IRQ-disabled region\n",
11633				func_id_name(func_id), func_id);
11634			return -EINVAL;
11635		}
11636	}
11637
11638	/* Track non-sleepable context for helpers. */
11639	if (!in_sleepable_context(env))
11640		env->insn_aux_data[insn_idx].non_sleepable = true;
11641
11642	meta.func_id = func_id;
11643	/* check args */
11644	for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
11645		err = check_func_arg(env, i, &meta, fn, insn_idx);
11646		if (err)
11647			return err;
11648	}
11649
11650	err = record_func_map(env, &meta, func_id, insn_idx);
11651	if (err)
11652		return err;
11653
11654	err = record_func_key(env, &meta, func_id, insn_idx);
11655	if (err)
11656		return err;
11657
11658	/* Mark slots with STACK_MISC in case of raw mode, stack offset
11659	 * is inferred from register state.
11660	 */
11661	for (i = 0; i < meta.access_size; i++) {
11662		err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B,
11663				       BPF_WRITE, -1, false, false);
11664		if (err)
11665			return err;
11666	}
11667
11668	regs = cur_regs(env);
11669
11670	if (meta.release_regno) {
11671		err = -EINVAL;
11672		if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1])) {
11673			err = unmark_stack_slots_dynptr(env, &regs[meta.release_regno]);
11674		} else if (func_id == BPF_FUNC_kptr_xchg && meta.ref_obj_id) {
11675			u32 ref_obj_id = meta.ref_obj_id;
11676			bool in_rcu = in_rcu_cs(env);
11677			struct bpf_func_state *state;
11678			struct bpf_reg_state *reg;
11679
11680			err = release_reference_nomark(env->cur_state, ref_obj_id);
11681			if (!err) {
11682				bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
11683					if (reg->ref_obj_id == ref_obj_id) {
11684						if (in_rcu && (reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) {
11685							reg->ref_obj_id = 0;
11686							reg->type &= ~MEM_ALLOC;
11687							reg->type |= MEM_RCU;
11688						} else {
11689							mark_reg_invalid(env, reg);
11690						}
11691					}
11692				}));
11693			}
11694		} else if (meta.ref_obj_id) {
11695			err = release_reference(env, meta.ref_obj_id);
11696		} else if (register_is_null(&regs[meta.release_regno])) {
11697			/* meta.ref_obj_id can only be 0 if register that is meant to be
11698			 * released is NULL, which must be > R0.
11699			 */
11700			err = 0;
11701		}
11702		if (err) {
11703			verbose(env, "func %s#%d reference has not been acquired before\n",
11704				func_id_name(func_id), func_id);
11705			return err;
11706		}
11707	}
11708
11709	switch (func_id) {
11710	case BPF_FUNC_tail_call:
11711		err = check_resource_leak(env, false, true, "tail_call");
11712		if (err)
11713			return err;
11714		break;
11715	case BPF_FUNC_get_local_storage:
11716		/* check that flags argument in get_local_storage(map, flags) is 0,
11717		 * this is required because get_local_storage() can't return an error.
11718		 */
11719		if (!register_is_null(&regs[BPF_REG_2])) {
11720			verbose(env, "get_local_storage() doesn't support non-zero flags\n");
11721			return -EINVAL;
11722		}
11723		break;
11724	case BPF_FUNC_for_each_map_elem:
11725		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
11726					 set_map_elem_callback_state);
11727		break;
11728	case BPF_FUNC_timer_set_callback:
11729		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
11730					 set_timer_callback_state);
11731		break;
11732	case BPF_FUNC_find_vma:
11733		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
11734					 set_find_vma_callback_state);
11735		break;
11736	case BPF_FUNC_snprintf:
11737		err = check_bpf_snprintf_call(env, regs);
11738		break;
11739	case BPF_FUNC_loop:
11740		update_loop_inline_state(env, meta.subprogno);
11741		/* Verifier relies on R1 value to determine if bpf_loop() iteration
11742		 * is finished, thus mark it precise.
11743		 */
11744		err = mark_chain_precision(env, BPF_REG_1);
11745		if (err)
11746			return err;
11747		if (cur_func(env)->callback_depth < regs[BPF_REG_1].umax_value) {
11748			err = push_callback_call(env, insn, insn_idx, meta.subprogno,
11749						 set_loop_callback_state);
11750		} else {
11751			cur_func(env)->callback_depth = 0;
11752			if (env->log.level & BPF_LOG_LEVEL2)
11753				verbose(env, "frame%d bpf_loop iteration limit reached\n",
11754					env->cur_state->curframe);
11755		}
11756		break;
11757	case BPF_FUNC_dynptr_from_mem:
11758		if (regs[BPF_REG_1].type != PTR_TO_MAP_VALUE) {
11759			verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n",
11760				reg_type_str(env, regs[BPF_REG_1].type));
11761			return -EACCES;
11762		}
11763		break;
11764	case BPF_FUNC_set_retval:
11765		if (prog_type == BPF_PROG_TYPE_LSM &&
11766		    env->prog->expected_attach_type == BPF_LSM_CGROUP) {
11767			if (!env->prog->aux->attach_func_proto->type) {
11768				/* Make sure programs that attach to void
11769				 * hooks don't try to modify return value.
11770				 */
11771				verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
11772				return -EINVAL;
11773			}
11774		}
11775		break;
11776	case BPF_FUNC_dynptr_data:
11777	{
11778		struct bpf_reg_state *reg;
11779		int id, ref_obj_id;
11780
11781		reg = get_dynptr_arg_reg(env, fn, regs);
11782		if (!reg)
11783			return -EFAULT;
11784
11785
11786		if (meta.dynptr_id) {
11787			verifier_bug(env, "meta.dynptr_id already set");
11788			return -EFAULT;
11789		}
11790		if (meta.ref_obj_id) {
11791			verifier_bug(env, "meta.ref_obj_id already set");
11792			return -EFAULT;
11793		}
11794
11795		id = dynptr_id(env, reg);
11796		if (id < 0) {
11797			verifier_bug(env, "failed to obtain dynptr id");
11798			return id;
11799		}
11800
11801		ref_obj_id = dynptr_ref_obj_id(env, reg);
11802		if (ref_obj_id < 0) {
11803			verifier_bug(env, "failed to obtain dynptr ref_obj_id");
11804			return ref_obj_id;
11805		}
11806
11807		meta.dynptr_id = id;
11808		meta.ref_obj_id = ref_obj_id;
11809
11810		break;
11811	}
11812	case BPF_FUNC_dynptr_write:
11813	{
11814		enum bpf_dynptr_type dynptr_type;
11815		struct bpf_reg_state *reg;
11816
11817		reg = get_dynptr_arg_reg(env, fn, regs);
11818		if (!reg)
11819			return -EFAULT;
11820
11821		dynptr_type = dynptr_get_type(env, reg);
11822		if (dynptr_type == BPF_DYNPTR_TYPE_INVALID)
11823			return -EFAULT;
11824
11825		if (dynptr_type == BPF_DYNPTR_TYPE_SKB ||
11826		    dynptr_type == BPF_DYNPTR_TYPE_SKB_META)
11827			/* this will trigger clear_all_pkt_pointers(), which will
11828			 * invalidate all dynptr slices associated with the skb
11829			 */
11830			changes_data = true;
11831
11832		break;
11833	}
11834	case BPF_FUNC_per_cpu_ptr:
11835	case BPF_FUNC_this_cpu_ptr:
11836	{
11837		struct bpf_reg_state *reg = &regs[BPF_REG_1];
11838		const struct btf_type *type;
11839
11840		if (reg->type & MEM_RCU) {
11841			type = btf_type_by_id(reg->btf, reg->btf_id);
11842			if (!type || !btf_type_is_struct(type)) {
11843				verbose(env, "Helper has invalid btf/btf_id in R1\n");
11844				return -EFAULT;
11845			}
11846			returns_cpu_specific_alloc_ptr = true;
11847			env->insn_aux_data[insn_idx].call_with_percpu_alloc_ptr = true;
11848		}
11849		break;
11850	}
11851	case BPF_FUNC_user_ringbuf_drain:
11852		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
11853					 set_user_ringbuf_callback_state);
11854		break;
11855	}
11856
11857	if (err)
11858		return err;
11859
11860	/* reset caller saved regs */
11861	for (i = 0; i < CALLER_SAVED_REGS; i++) {
11862		mark_reg_not_init(env, regs, caller_saved[i]);
11863		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
11864	}
11865
11866	/* helper call returns 64-bit value. */
11867	regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
11868
11869	/* update return register (already marked as written above) */
11870	ret_type = fn->ret_type;
11871	ret_flag = type_flag(ret_type);
11872
11873	switch (base_type(ret_type)) {
11874	case RET_INTEGER:
11875		/* sets type to SCALAR_VALUE */
11876		mark_reg_unknown(env, regs, BPF_REG_0);
11877		break;
11878	case RET_VOID:
11879		regs[BPF_REG_0].type = NOT_INIT;
11880		break;
11881	case RET_PTR_TO_MAP_VALUE:
11882		/* There is no offset yet applied, variable or fixed */
11883		mark_reg_known_zero(env, regs, BPF_REG_0);
11884		/* remember map_ptr, so that check_map_access()
11885		 * can check 'value_size' boundary of memory access
11886		 * to map element returned from bpf_map_lookup_elem()
11887		 */
11888		if (meta.map_ptr == NULL) {
11889			verifier_bug(env, "unexpected null map_ptr");
11890			return -EFAULT;
11891		}
11892
11893		if (func_id == BPF_FUNC_map_lookup_elem &&
11894		    can_elide_value_nullness(meta.map_ptr->map_type) &&
11895		    meta.const_map_key >= 0 &&
11896		    meta.const_map_key < meta.map_ptr->max_entries)
11897			ret_flag &= ~PTR_MAYBE_NULL;
11898
11899		regs[BPF_REG_0].map_ptr = meta.map_ptr;
11900		regs[BPF_REG_0].map_uid = meta.map_uid;
11901		regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag;
11902		if (!type_may_be_null(ret_flag) &&
11903		    btf_record_has_field(meta.map_ptr->record, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK)) {
11904			regs[BPF_REG_0].id = ++env->id_gen;
11905		}
11906		break;
11907	case RET_PTR_TO_SOCKET:
11908		mark_reg_known_zero(env, regs, BPF_REG_0);
11909		regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag;
11910		break;
11911	case RET_PTR_TO_SOCK_COMMON:
11912		mark_reg_known_zero(env, regs, BPF_REG_0);
11913		regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag;
11914		break;
11915	case RET_PTR_TO_TCP_SOCK:
11916		mark_reg_known_zero(env, regs, BPF_REG_0);
11917		regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag;
11918		break;
11919	case RET_PTR_TO_MEM:
11920		mark_reg_known_zero(env, regs, BPF_REG_0);
11921		regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
11922		regs[BPF_REG_0].mem_size = meta.mem_size;
11923		break;
11924	case RET_PTR_TO_MEM_OR_BTF_ID:
11925	{
11926		const struct btf_type *t;
11927
11928		mark_reg_known_zero(env, regs, BPF_REG_0);
11929		t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL);
11930		if (!btf_type_is_struct(t)) {
11931			u32 tsize;
11932			const struct btf_type *ret;
11933			const char *tname;
11934
11935			/* resolve the type size of ksym. */
11936			ret = btf_resolve_size(meta.ret_btf, t, &tsize);
11937			if (IS_ERR(ret)) {
11938				tname = btf_name_by_offset(meta.ret_btf, t->name_off);
11939				verbose(env, "unable to resolve the size of type '%s': %ld\n",
11940					tname, PTR_ERR(ret));
11941				return -EINVAL;
11942			}
11943			regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
11944			regs[BPF_REG_0].mem_size = tsize;
11945		} else {
11946			if (returns_cpu_specific_alloc_ptr) {
11947				regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC | MEM_RCU;
11948			} else {
11949				/* MEM_RDONLY may be carried from ret_flag, but it
11950				 * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise
11951				 * it will confuse the check of PTR_TO_BTF_ID in
11952				 * check_mem_access().
11953				 */
11954				ret_flag &= ~MEM_RDONLY;
11955				regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag;
11956			}
11957
11958			regs[BPF_REG_0].btf = meta.ret_btf;
11959			regs[BPF_REG_0].btf_id = meta.ret_btf_id;
11960		}
11961		break;
11962	}
11963	case RET_PTR_TO_BTF_ID:
11964	{
11965		struct btf *ret_btf;
11966		int ret_btf_id;
11967
11968		mark_reg_known_zero(env, regs, BPF_REG_0);
11969		regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag;
11970		if (func_id == BPF_FUNC_kptr_xchg) {
11971			ret_btf = meta.kptr_field->kptr.btf;
11972			ret_btf_id = meta.kptr_field->kptr.btf_id;
11973			if (!btf_is_kernel(ret_btf)) {
11974				regs[BPF_REG_0].type |= MEM_ALLOC;
11975				if (meta.kptr_field->type == BPF_KPTR_PERCPU)
11976					regs[BPF_REG_0].type |= MEM_PERCPU;
11977			}
11978		} else {
11979			if (fn->ret_btf_id == BPF_PTR_POISON) {
11980				verifier_bug(env, "func %s has non-overwritten BPF_PTR_POISON return type",
11981					     func_id_name(func_id));
11982				return -EFAULT;
11983			}
11984			ret_btf = btf_vmlinux;
11985			ret_btf_id = *fn->ret_btf_id;
11986		}
11987		if (ret_btf_id == 0) {
11988			verbose(env, "invalid return type %u of func %s#%d\n",
11989				base_type(ret_type), func_id_name(func_id),
11990				func_id);
11991			return -EINVAL;
11992		}
11993		regs[BPF_REG_0].btf = ret_btf;
11994		regs[BPF_REG_0].btf_id = ret_btf_id;
11995		break;
11996	}
11997	default:
11998		verbose(env, "unknown return type %u of func %s#%d\n",
11999			base_type(ret_type), func_id_name(func_id), func_id);
12000		return -EINVAL;
12001	}
12002
12003	if (type_may_be_null(regs[BPF_REG_0].type))
12004		regs[BPF_REG_0].id = ++env->id_gen;
12005
12006	if (helper_multiple_ref_obj_use(func_id, meta.map_ptr)) {
12007		verifier_bug(env, "func %s#%d sets ref_obj_id more than once",
12008			     func_id_name(func_id), func_id);
12009		return -EFAULT;
12010	}
12011
12012	if (is_dynptr_ref_function(func_id))
12013		regs[BPF_REG_0].dynptr_id = meta.dynptr_id;
12014
12015	if (is_ptr_cast_function(func_id) || is_dynptr_ref_function(func_id)) {
12016		/* For release_reference() */
12017		regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
12018	} else if (is_acquire_function(func_id, meta.map_ptr)) {
12019		int id = acquire_reference(env, insn_idx);
12020
12021		if (id < 0)
12022			return id;
12023		/* For mark_ptr_or_null_reg() */
12024		regs[BPF_REG_0].id = id;
12025		/* For release_reference() */
12026		regs[BPF_REG_0].ref_obj_id = id;
12027	}
12028
12029	err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta);
12030	if (err)
12031		return err;
12032
12033	err = check_map_func_compatibility(env, meta.map_ptr, func_id);
12034	if (err)
12035		return err;
12036
12037	if ((func_id == BPF_FUNC_get_stack ||
12038	     func_id == BPF_FUNC_get_task_stack) &&
12039	    !env->prog->has_callchain_buf) {
12040		const char *err_str;
12041
12042#ifdef CONFIG_PERF_EVENTS
12043		err = get_callchain_buffers(sysctl_perf_event_max_stack);
12044		err_str = "cannot get callchain buffer for func %s#%d\n";
12045#else
12046		err = -ENOTSUPP;
12047		err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n";
12048#endif
12049		if (err) {
12050			verbose(env, err_str, func_id_name(func_id), func_id);
12051			return err;
12052		}
12053
12054		env->prog->has_callchain_buf = true;
12055	}
12056
12057	if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack)
12058		env->prog->call_get_stack = true;
12059
12060	if (func_id == BPF_FUNC_get_func_ip) {
12061		if (check_get_func_ip(env))
12062			return -ENOTSUPP;
12063		env->prog->call_get_func_ip = true;
12064	}
12065
12066	if (func_id == BPF_FUNC_tail_call) {
12067		if (env->cur_state->curframe) {
12068			struct bpf_verifier_state *branch;
12069
12070			mark_reg_scratched(env, BPF_REG_0);
12071			branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);
12072			if (IS_ERR(branch))
12073				return PTR_ERR(branch);
12074			clear_all_pkt_pointers(env);
12075			mark_reg_unknown(env, regs, BPF_REG_0);
12076			err = prepare_func_exit(env, &env->insn_idx);
12077			if (err)
12078				return err;
12079			env->insn_idx--;
12080		} else {
12081			changes_data = false;
12082		}
12083	}
12084
12085	if (changes_data)
12086		clear_all_pkt_pointers(env);
12087	return 0;
12088}
12089
12090/* mark_btf_func_reg_size() is used when the reg size is determined by
12091 * the BTF func_proto's return value size and argument.
12092 */
12093static void __mark_btf_func_reg_size(struct bpf_verifier_env *env, struct bpf_reg_state *regs,
12094				     u32 regno, size_t reg_size)
12095{
12096	struct bpf_reg_state *reg = &regs[regno];
12097
12098	if (regno == BPF_REG_0) {
12099		/* Function return value */
12100		reg->subreg_def = reg_size == sizeof(u64) ?
12101			DEF_NOT_SUBREG : env->insn_idx + 1;
12102	} else if (reg_size == sizeof(u64)) {
12103		/* Function argument */
12104		mark_insn_zext(env, reg);
12105	}
12106}
12107
12108static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,
12109				   size_t reg_size)
12110{
12111	return __mark_btf_func_reg_size(env, cur_regs(env), regno, reg_size);
12112}
12113
12114static bool is_kfunc_acquire(struct bpf_kfunc_call_arg_meta *meta)
12115{
12116	return meta->kfunc_flags & KF_ACQUIRE;
12117}
12118
12119static bool is_kfunc_release(struct bpf_kfunc_call_arg_meta *meta)
12120{
12121	return meta->kfunc_flags & KF_RELEASE;
12122}
12123
12124static bool is_kfunc_trusted_args(struct bpf_kfunc_call_arg_meta *meta)
12125{
12126	return (meta->kfunc_flags & KF_TRUSTED_ARGS) || is_kfunc_release(meta);
12127}
12128
12129static bool is_kfunc_sleepable(struct bpf_kfunc_call_arg_meta *meta)
12130{
12131	return meta->kfunc_flags & KF_SLEEPABLE;
12132}
12133
12134static bool is_kfunc_destructive(struct bpf_kfunc_call_arg_meta *meta)
12135{
12136	return meta->kfunc_flags & KF_DESTRUCTIVE;
12137}
12138
12139static bool is_kfunc_rcu(struct bpf_kfunc_call_arg_meta *meta)
12140{
12141	return meta->kfunc_flags & KF_RCU;
12142}
12143
12144static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta)
12145{
12146	return meta->kfunc_flags & KF_RCU_PROTECTED;
12147}
12148
12149static bool is_kfunc_arg_mem_size(const struct btf *btf,
12150				  const struct btf_param *arg,
12151				  const struct bpf_reg_state *reg)
12152{
12153	const struct btf_type *t;
12154
12155	t = btf_type_skip_modifiers(btf, arg->type, NULL);
12156	if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE)
12157		return false;
12158
12159	return btf_param_match_suffix(btf, arg, "__sz");
12160}
12161
12162static bool is_kfunc_arg_const_mem_size(const struct btf *btf,
12163					const struct btf_param *arg,
12164					const struct bpf_reg_state *reg)
12165{
12166	const struct btf_type *t;
12167
12168	t = btf_type_skip_modifiers(btf, arg->type, NULL);
12169	if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE)
12170		return false;
12171
12172	return btf_param_match_suffix(btf, arg, "__szk");
12173}
12174
12175static bool is_kfunc_arg_optional(const struct btf *btf, const struct btf_param *arg)
12176{
12177	return btf_param_match_suffix(btf, arg, "__opt");
12178}
12179
12180static bool is_kfunc_arg_constant(const struct btf *btf, const struct btf_param *arg)
12181{
12182	return btf_param_match_suffix(btf, arg, "__k");
12183}
12184
12185static bool is_kfunc_arg_ignore(const struct btf *btf, const struct btf_param *arg)
12186{
12187	return btf_param_match_suffix(btf, arg, "__ign");
12188}
12189
12190static bool is_kfunc_arg_map(const struct btf *btf, const struct btf_param *arg)
12191{
12192	return btf_param_match_suffix(btf, arg, "__map");
12193}
12194
12195static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg)
12196{
12197	return btf_param_match_suffix(btf, arg, "__alloc");
12198}
12199
12200static bool is_kfunc_arg_uninit(const struct btf *btf, const struct btf_param *arg)
12201{
12202	return btf_param_match_suffix(btf, arg, "__uninit");
12203}
12204
12205static bool is_kfunc_arg_refcounted_kptr(const struct btf *btf, const struct btf_param *arg)
12206{
12207	return btf_param_match_suffix(btf, arg, "__refcounted_kptr");
12208}
12209
12210static bool is_kfunc_arg_nullable(const struct btf *btf, const struct btf_param *arg)
12211{
12212	return btf_param_match_suffix(btf, arg, "__nullable");
12213}
12214
12215static bool is_kfunc_arg_const_str(const struct btf *btf, const struct btf_param *arg)
12216{
12217	return btf_param_match_suffix(btf, arg, "__str");
12218}
12219
12220static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param *arg)
12221{
12222	return btf_param_match_suffix(btf, arg, "__irq_flag");
12223}
12224
12225static bool is_kfunc_arg_prog(const struct btf *btf, const struct btf_param *arg)
12226{
12227	return btf_param_match_suffix(btf, arg, "__prog");
12228}
12229
12230static bool is_kfunc_arg_scalar_with_name(const struct btf *btf,
12231					  const struct btf_param *arg,
12232					  const char *name)
12233{
12234	int len, target_len = strlen(name);
12235	const char *param_name;
12236
12237	param_name = btf_name_by_offset(btf, arg->name_off);
12238	if (str_is_empty(param_name))
12239		return false;
12240	len = strlen(param_name);
12241	if (len != target_len)
12242		return false;
12243	if (strcmp(param_name, name))
12244		return false;
12245
12246	return true;
12247}
12248
12249enum {
12250	KF_ARG_DYNPTR_ID,
12251	KF_ARG_LIST_HEAD_ID,
12252	KF_ARG_LIST_NODE_ID,
12253	KF_ARG_RB_ROOT_ID,
12254	KF_ARG_RB_NODE_ID,
12255	KF_ARG_WORKQUEUE_ID,
12256	KF_ARG_RES_SPIN_LOCK_ID,
12257	KF_ARG_TASK_WORK_ID,
12258};
12259
12260BTF_ID_LIST(kf_arg_btf_ids)
12261BTF_ID(struct, bpf_dynptr)
12262BTF_ID(struct, bpf_list_head)
12263BTF_ID(struct, bpf_list_node)
12264BTF_ID(struct, bpf_rb_root)
12265BTF_ID(struct, bpf_rb_node)
12266BTF_ID(struct, bpf_wq)
12267BTF_ID(struct, bpf_res_spin_lock)
12268BTF_ID(struct, bpf_task_work)
12269
12270static bool __is_kfunc_ptr_arg_type(const struct btf *btf,
12271				    const struct btf_param *arg, int type)
12272{
12273	const struct btf_type *t;
12274	u32 res_id;
12275
12276	t = btf_type_skip_modifiers(btf, arg->type, NULL);
12277	if (!t)
12278		return false;
12279	if (!btf_type_is_ptr(t))
12280		return false;
12281	t = btf_type_skip_modifiers(btf, t->type, &res_id);
12282	if (!t)
12283		return false;
12284	return btf_types_are_same(btf, res_id, btf_vmlinux, kf_arg_btf_ids[type]);
12285}
12286
12287static bool is_kfunc_arg_dynptr(const struct btf *btf, const struct btf_param *arg)
12288{
12289	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_DYNPTR_ID);
12290}
12291
12292static bool is_kfunc_arg_list_head(const struct btf *btf, const struct btf_param *arg)
12293{
12294	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_HEAD_ID);
12295}
12296
12297static bool is_kfunc_arg_list_node(const struct btf *btf, const struct btf_param *arg)
12298{
12299	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_NODE_ID);
12300}
12301
12302static bool is_kfunc_arg_rbtree_root(const struct btf *btf, const struct btf_param *arg)
12303{
12304	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_ROOT_ID);
12305}
12306
12307static bool is_kfunc_arg_rbtree_node(const struct btf *btf, const struct btf_param *arg)
12308{
12309	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_NODE_ID);
12310}
12311
12312static bool is_kfunc_arg_wq(const struct btf *btf, const struct btf_param *arg)
12313{
12314	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_WORKQUEUE_ID);
12315}
12316
12317static bool is_kfunc_arg_task_work(const struct btf *btf, const struct btf_param *arg)
12318{
12319	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_TASK_WORK_ID);
12320}
12321
12322static bool is_kfunc_arg_res_spin_lock(const struct btf *btf, const struct btf_param *arg)
12323{
12324	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RES_SPIN_LOCK_ID);
12325}
12326
12327static bool is_rbtree_node_type(const struct btf_type *t)
12328{
12329	return t == btf_type_by_id(btf_vmlinux, kf_arg_btf_ids[KF_ARG_RB_NODE_ID]);
12330}
12331
12332static bool is_list_node_type(const struct btf_type *t)
12333{
12334	return t == btf_type_by_id(btf_vmlinux, kf_arg_btf_ids[KF_ARG_LIST_NODE_ID]);
12335}
12336
12337static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf,
12338				  const struct btf_param *arg)
12339{
12340	const struct btf_type *t;
12341
12342	t = btf_type_resolve_func_ptr(btf, arg->type, NULL);
12343	if (!t)
12344		return false;
12345
12346	return true;
12347}
12348
12349/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */
12350static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env,
12351					const struct btf *btf,
12352					const struct btf_type *t, int rec)
12353{
12354	const struct btf_type *member_type;
12355	const struct btf_member *member;
12356	u32 i;
12357
12358	if (!btf_type_is_struct(t))
12359		return false;
12360
12361	for_each_member(i, t, member) {
12362		const struct btf_array *array;
12363
12364		member_type = btf_type_skip_modifiers(btf, member->type, NULL);
12365		if (btf_type_is_struct(member_type)) {
12366			if (rec >= 3) {
12367				verbose(env, "max struct nesting depth exceeded\n");
12368				return false;
12369			}
12370			if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1))
12371				return false;
12372			continue;
12373		}
12374		if (btf_type_is_array(member_type)) {
12375			array = btf_array(member_type);
12376			if (!array->nelems)
12377				return false;
12378			member_type = btf_type_skip_modifiers(btf, array->type, NULL);
12379			if (!btf_type_is_scalar(member_type))
12380				return false;
12381			continue;
12382		}
12383		if (!btf_type_is_scalar(member_type))
12384			return false;
12385	}
12386	return true;
12387}
12388
12389enum kfunc_ptr_arg_type {
12390	KF_ARG_PTR_TO_CTX,
12391	KF_ARG_PTR_TO_ALLOC_BTF_ID,    /* Allocated object */
12392	KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */
12393	KF_ARG_PTR_TO_DYNPTR,
12394	KF_ARG_PTR_TO_ITER,
12395	KF_ARG_PTR_TO_LIST_HEAD,
12396	KF_ARG_PTR_TO_LIST_NODE,
12397	KF_ARG_PTR_TO_BTF_ID,	       /* Also covers reg2btf_ids conversions */
12398	KF_ARG_PTR_TO_MEM,
12399	KF_ARG_PTR_TO_MEM_SIZE,	       /* Size derived from next argument, skip it */
12400	KF_ARG_PTR_TO_CALLBACK,
12401	KF_ARG_PTR_TO_RB_ROOT,
12402	KF_ARG_PTR_TO_RB_NODE,
12403	KF_ARG_PTR_TO_NULL,
12404	KF_ARG_PTR_TO_CONST_STR,
12405	KF_ARG_PTR_TO_MAP,
12406	KF_ARG_PTR_TO_WORKQUEUE,
12407	KF_ARG_PTR_TO_IRQ_FLAG,
12408	KF_ARG_PTR_TO_RES_SPIN_LOCK,
12409	KF_ARG_PTR_TO_TASK_WORK,
12410};
12411
12412enum special_kfunc_type {
12413	KF_bpf_obj_new_impl,
12414	KF_bpf_obj_drop_impl,
12415	KF_bpf_refcount_acquire_impl,
12416	KF_bpf_list_push_front_impl,
12417	KF_bpf_list_push_back_impl,
12418	KF_bpf_list_pop_front,
12419	KF_bpf_list_pop_back,
12420	KF_bpf_list_front,
12421	KF_bpf_list_back,
12422	KF_bpf_cast_to_kern_ctx,
12423	KF_bpf_rdonly_cast,
12424	KF_bpf_rcu_read_lock,
12425	KF_bpf_rcu_read_unlock,
12426	KF_bpf_rbtree_remove,
12427	KF_bpf_rbtree_add_impl,
12428	KF_bpf_rbtree_first,
12429	KF_bpf_rbtree_root,
12430	KF_bpf_rbtree_left,
12431	KF_bpf_rbtree_right,
12432	KF_bpf_dynptr_from_skb,
12433	KF_bpf_dynptr_from_xdp,
12434	KF_bpf_dynptr_from_skb_meta,
12435	KF_bpf_xdp_pull_data,
12436	KF_bpf_dynptr_slice,
12437	KF_bpf_dynptr_slice_rdwr,
12438	KF_bpf_dynptr_clone,
12439	KF_bpf_percpu_obj_new_impl,
12440	KF_bpf_percpu_obj_drop_impl,
12441	KF_bpf_throw,
12442	KF_bpf_wq_set_callback_impl,
12443	KF_bpf_preempt_disable,
12444	KF_bpf_preempt_enable,
12445	KF_bpf_iter_css_task_new,
12446	KF_bpf_session_cookie,
12447	KF_bpf_get_kmem_cache,
12448	KF_bpf_local_irq_save,
12449	KF_bpf_local_irq_restore,
12450	KF_bpf_iter_num_new,
12451	KF_bpf_iter_num_next,
12452	KF_bpf_iter_num_destroy,
12453	KF_bpf_set_dentry_xattr,
12454	KF_bpf_remove_dentry_xattr,
12455	KF_bpf_res_spin_lock,
12456	KF_bpf_res_spin_unlock,
12457	KF_bpf_res_spin_lock_irqsave,
12458	KF_bpf_res_spin_unlock_irqrestore,
12459	KF_bpf_dynptr_from_file,
12460	KF_bpf_dynptr_file_discard,
12461	KF___bpf_trap,
12462	KF_bpf_task_work_schedule_signal_impl,
12463	KF_bpf_task_work_schedule_resume_impl,
12464};
12465
12466BTF_ID_LIST(special_kfunc_list)
12467BTF_ID(func, bpf_obj_new_impl)
12468BTF_ID(func, bpf_obj_drop_impl)
12469BTF_ID(func, bpf_refcount_acquire_impl)
12470BTF_ID(func, bpf_list_push_front_impl)
12471BTF_ID(func, bpf_list_push_back_impl)
12472BTF_ID(func, bpf_list_pop_front)
12473BTF_ID(func, bpf_list_pop_back)
12474BTF_ID(func, bpf_list_front)
12475BTF_ID(func, bpf_list_back)
12476BTF_ID(func, bpf_cast_to_kern_ctx)
12477BTF_ID(func, bpf_rdonly_cast)
12478BTF_ID(func, bpf_rcu_read_lock)
12479BTF_ID(func, bpf_rcu_read_unlock)
12480BTF_ID(func, bpf_rbtree_remove)
12481BTF_ID(func, bpf_rbtree_add_impl)
12482BTF_ID(func, bpf_rbtree_first)
12483BTF_ID(func, bpf_rbtree_root)
12484BTF_ID(func, bpf_rbtree_left)
12485BTF_ID(func, bpf_rbtree_right)
12486#ifdef CONFIG_NET
12487BTF_ID(func, bpf_dynptr_from_skb)
12488BTF_ID(func, bpf_dynptr_from_xdp)
12489BTF_ID(func, bpf_dynptr_from_skb_meta)
12490BTF_ID(func, bpf_xdp_pull_data)
12491#else
12492BTF_ID_UNUSED
12493BTF_ID_UNUSED
12494BTF_ID_UNUSED
12495BTF_ID_UNUSED
12496#endif
12497BTF_ID(func, bpf_dynptr_slice)
12498BTF_ID(func, bpf_dynptr_slice_rdwr)
12499BTF_ID(func, bpf_dynptr_clone)
12500BTF_ID(func, bpf_percpu_obj_new_impl)
12501BTF_ID(func, bpf_percpu_obj_drop_impl)
12502BTF_ID(func, bpf_throw)
12503BTF_ID(func, bpf_wq_set_callback_impl)
12504BTF_ID(func, bpf_preempt_disable)
12505BTF_ID(func, bpf_preempt_enable)
12506#ifdef CONFIG_CGROUPS
12507BTF_ID(func, bpf_iter_css_task_new)
12508#else
12509BTF_ID_UNUSED
12510#endif
12511#ifdef CONFIG_BPF_EVENTS
12512BTF_ID(func, bpf_session_cookie)
12513#else
12514BTF_ID_UNUSED
12515#endif
12516BTF_ID(func, bpf_get_kmem_cache)
12517BTF_ID(func, bpf_local_irq_save)
12518BTF_ID(func, bpf_local_irq_restore)
12519BTF_ID(func, bpf_iter_num_new)
12520BTF_ID(func, bpf_iter_num_next)
12521BTF_ID(func, bpf_iter_num_destroy)
12522#ifdef CONFIG_BPF_LSM
12523BTF_ID(func, bpf_set_dentry_xattr)
12524BTF_ID(func, bpf_remove_dentry_xattr)
12525#else
12526BTF_ID_UNUSED
12527BTF_ID_UNUSED
12528#endif
12529BTF_ID(func, bpf_res_spin_lock)
12530BTF_ID(func, bpf_res_spin_unlock)
12531BTF_ID(func, bpf_res_spin_lock_irqsave)
12532BTF_ID(func, bpf_res_spin_unlock_irqrestore)
12533BTF_ID(func, bpf_dynptr_from_file)
12534BTF_ID(func, bpf_dynptr_file_discard)
12535BTF_ID(func, __bpf_trap)
12536BTF_ID(func, bpf_task_work_schedule_signal_impl)
12537BTF_ID(func, bpf_task_work_schedule_resume_impl)
12538
12539static bool is_task_work_add_kfunc(u32 func_id)
12540{
12541	return func_id == special_kfunc_list[KF_bpf_task_work_schedule_signal_impl] ||
12542	       func_id == special_kfunc_list[KF_bpf_task_work_schedule_resume_impl];
12543}
12544
12545static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta)
12546{
12547	if (meta->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl] &&
12548	    meta->arg_owning_ref) {
12549		return false;
12550	}
12551
12552	return meta->kfunc_flags & KF_RET_NULL;
12553}
12554
12555static bool is_kfunc_bpf_rcu_read_lock(struct bpf_kfunc_call_arg_meta *meta)
12556{
12557	return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock];
12558}
12559
12560static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_kfunc_call_arg_meta *meta)
12561{
12562	return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock];
12563}
12564
12565static bool is_kfunc_bpf_preempt_disable(struct bpf_kfunc_call_arg_meta *meta)
12566{
12567	return meta->func_id == special_kfunc_list[KF_bpf_preempt_disable];
12568}
12569
12570static bool is_kfunc_bpf_preempt_enable(struct bpf_kfunc_call_arg_meta *meta)
12571{
12572	return meta->func_id == special_kfunc_list[KF_bpf_preempt_enable];
12573}
12574
12575static bool is_kfunc_pkt_changing(struct bpf_kfunc_call_arg_meta *meta)
12576{
12577	return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data];
12578}
12579
12580static enum kfunc_ptr_arg_type
12581get_kfunc_ptr_arg_type(struct bpf_verifier_env *env,
12582		       struct bpf_kfunc_call_arg_meta *meta,
12583		       const struct btf_type *t, const struct btf_type *ref_t,
12584		       const char *ref_tname, const struct btf_param *args,
12585		       int argno, int nargs)
12586{
12587	u32 regno = argno + 1;
12588	struct bpf_reg_state *regs = cur_regs(env);
12589	struct bpf_reg_state *reg = &regs[regno];
12590	bool arg_mem_size = false;
12591
12592	if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx])
12593		return KF_ARG_PTR_TO_CTX;
12594
12595	/* In this function, we verify the kfunc's BTF as per the argument type,
12596	 * leaving the rest of the verification with respect to the register
12597	 * type to our caller. When a set of conditions hold in the BTF type of
12598	 * arguments, we resolve it to a known kfunc_ptr_arg_type.
12599	 */
12600	if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno))
12601		return KF_ARG_PTR_TO_CTX;
12602
12603	if (is_kfunc_arg_nullable(meta->btf, &args[argno]) && register_is_null(reg))
12604		return KF_ARG_PTR_TO_NULL;
12605
12606	if (is_kfunc_arg_alloc_obj(meta->btf, &args[argno]))
12607		return KF_ARG_PTR_TO_ALLOC_BTF_ID;
12608
12609	if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[argno]))
12610		return KF_ARG_PTR_TO_REFCOUNTED_KPTR;
12611
12612	if (is_kfunc_arg_dynptr(meta->btf, &args[argno]))
12613		return KF_ARG_PTR_TO_DYNPTR;
12614
12615	if (is_kfunc_arg_iter(meta, argno, &args[argno]))
12616		return KF_ARG_PTR_TO_ITER;
12617
12618	if (is_kfunc_arg_list_head(meta->btf, &args[argno]))
12619		return KF_ARG_PTR_TO_LIST_HEAD;
12620
12621	if (is_kfunc_arg_list_node(meta->btf, &args[argno]))
12622		return KF_ARG_PTR_TO_LIST_NODE;
12623
12624	if (is_kfunc_arg_rbtree_root(meta->btf, &args[argno]))
12625		return KF_ARG_PTR_TO_RB_ROOT;
12626
12627	if (is_kfunc_arg_rbtree_node(meta->btf, &args[argno]))
12628		return KF_ARG_PTR_TO_RB_NODE;
12629
12630	if (is_kfunc_arg_const_str(meta->btf, &args[argno]))
12631		return KF_ARG_PTR_TO_CONST_STR;
12632
12633	if (is_kfunc_arg_map(meta->btf, &args[argno]))
12634		return KF_ARG_PTR_TO_MAP;
12635
12636	if (is_kfunc_arg_wq(meta->btf, &args[argno]))
12637		return KF_ARG_PTR_TO_WORKQUEUE;
12638
12639	if (is_kfunc_arg_task_work(meta->btf, &args[argno]))
12640		return KF_ARG_PTR_TO_TASK_WORK;
12641
12642	if (is_kfunc_arg_irq_flag(meta->btf, &args[argno]))
12643		return KF_ARG_PTR_TO_IRQ_FLAG;
12644
12645	if (is_kfunc_arg_res_spin_lock(meta->btf, &args[argno]))
12646		return KF_ARG_PTR_TO_RES_SPIN_LOCK;
12647
12648	if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) {
12649		if (!btf_type_is_struct(ref_t)) {
12650			verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n",
12651				meta->func_name, argno, btf_type_str(ref_t), ref_tname);
12652			return -EINVAL;
12653		}
12654		return KF_ARG_PTR_TO_BTF_ID;
12655	}
12656
12657	if (is_kfunc_arg_callback(env, meta->btf, &args[argno]))
12658		return KF_ARG_PTR_TO_CALLBACK;
12659
12660	if (argno + 1 < nargs &&
12661	    (is_kfunc_arg_mem_size(meta->btf, &args[argno + 1], &regs[regno + 1]) ||
12662	     is_kfunc_arg_const_mem_size(meta->btf, &args[argno + 1], &regs[regno + 1])))
12663		arg_mem_size = true;
12664
12665	/* This is the catch all argument type of register types supported by
12666	 * check_helper_mem_access. However, we only allow when argument type is
12667	 * pointer to scalar, or struct composed (recursively) of scalars. When
12668	 * arg_mem_size is true, the pointer can be void *.
12669	 */
12670	if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) &&
12671	    (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) {
12672		verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n",
12673			argno, btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : "");
12674		return -EINVAL;
12675	}
12676	return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM;
12677}
12678
12679static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,
12680					struct bpf_reg_state *reg,
12681					const struct btf_type *ref_t,
12682					const char *ref_tname, u32 ref_id,
12683					struct bpf_kfunc_call_arg_meta *meta,
12684					int argno)
12685{
12686	const struct btf_type *reg_ref_t;
12687	bool strict_type_match = false;
12688	const struct btf *reg_btf;
12689	const char *reg_ref_tname;
12690	bool taking_projection;
12691	bool struct_same;
12692	u32 reg_ref_id;
12693
12694	if (base_type(reg->type) == PTR_TO_BTF_ID) {
12695		reg_btf = reg->btf;
12696		reg_ref_id = reg->btf_id;
12697	} else {
12698		reg_btf = btf_vmlinux;
12699		reg_ref_id = *reg2btf_ids[base_type(reg->type)];
12700	}
12701
12702	/* Enforce strict type matching for calls to kfuncs that are acquiring
12703	 * or releasing a reference, or are no-cast aliases. We do _not_
12704	 * enforce strict matching for plain KF_TRUSTED_ARGS kfuncs by default,
12705	 * as we want to enable BPF programs to pass types that are bitwise
12706	 * equivalent without forcing them to explicitly cast with something
12707	 * like bpf_cast_to_kern_ctx().
12708	 *
12709	 * For example, say we had a type like the following:
12710	 *
12711	 * struct bpf_cpumask {
12712	 *	cpumask_t cpumask;
12713	 *	refcount_t usage;
12714	 * };
12715	 *
12716	 * Note that as specified in <linux/cpumask.h>, cpumask_t is typedef'ed
12717	 * to a struct cpumask, so it would be safe to pass a struct
12718	 * bpf_cpumask * to a kfunc expecting a struct cpumask *.
12719	 *
12720	 * The philosophy here is similar to how we allow scalars of different
12721	 * types to be passed to kfuncs as long as the size is the same. The
12722	 * only difference here is that we're simply allowing
12723	 * btf_struct_ids_match() to walk the struct at the 0th offset, and
12724	 * resolve types.
12725	 */
12726	if ((is_kfunc_release(meta) && reg->ref_obj_id) ||
12727	    btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id))
12728		strict_type_match = true;
12729
12730	WARN_ON_ONCE(is_kfunc_release(meta) &&
12731		     (reg->off || !tnum_is_const(reg->var_off) ||
12732		      reg->var_off.value));
12733
12734	reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, &reg_ref_id);
12735	reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off);
12736	struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->off, meta->btf, ref_id, strict_type_match);
12737	/* If kfunc is accepting a projection type (ie. __sk_buff), it cannot
12738	 * actually use it -- it must cast to the underlying type. So we allow
12739	 * caller to pass in the underlying type.
12740	 */
12741	taking_projection = btf_is_projection_of(ref_tname, reg_ref_tname);
12742	if (!taking_projection && !struct_same) {
12743		verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n",
12744			meta->func_name, argno, btf_type_str(ref_t), ref_tname, argno + 1,
12745			btf_type_str(reg_ref_t), reg_ref_tname);
12746		return -EINVAL;
12747	}
12748	return 0;
12749}
12750
12751static int process_irq_flag(struct bpf_verifier_env *env, int regno,
12752			     struct bpf_kfunc_call_arg_meta *meta)
12753{
12754	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
12755	int err, kfunc_class = IRQ_NATIVE_KFUNC;
12756	bool irq_save;
12757
12758	if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_save] ||
12759	    meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) {
12760		irq_save = true;
12761		if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])
12762			kfunc_class = IRQ_LOCK_KFUNC;
12763	} else if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_restore] ||
12764		   meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) {
12765		irq_save = false;
12766		if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])
12767			kfunc_class = IRQ_LOCK_KFUNC;
12768	} else {
12769		verifier_bug(env, "unknown irq flags kfunc");
12770		return -EFAULT;
12771	}
12772
12773	if (irq_save) {
12774		if (!is_irq_flag_reg_valid_uninit(env, reg)) {
12775			verbose(env, "expected uninitialized irq flag as arg#%d\n", regno - 1);
12776			return -EINVAL;
12777		}
12778
12779		err = check_mem_access(env, env->insn_idx, regno, 0, BPF_DW, BPF_WRITE, -1, false, false);
12780		if (err)
12781			return err;
12782
12783		err = mark_stack_slot_irq_flag(env, meta, reg, env->insn_idx, kfunc_class);
12784		if (err)
12785			return err;
12786	} else {
12787		err = is_irq_flag_reg_valid_init(env, reg);
12788		if (err) {
12789			verbose(env, "expected an initialized irq flag as arg#%d\n", regno - 1);
12790			return err;
12791		}
12792
12793		err = mark_irq_flag_read(env, reg);
12794		if (err)
12795			return err;
12796
12797		err = unmark_stack_slot_irq_flag(env, reg, kfunc_class);
12798		if (err)
12799			return err;
12800	}
12801	return 0;
12802}
12803
12804
12805static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
12806{
12807	struct btf_record *rec = reg_btf_record(reg);
12808
12809	if (!env->cur_state->active_locks) {
12810		verifier_bug(env, "%s w/o active lock", __func__);
12811		return -EFAULT;
12812	}
12813
12814	if (type_flag(reg->type) & NON_OWN_REF) {
12815		verifier_bug(env, "NON_OWN_REF already set");
12816		return -EFAULT;
12817	}
12818
12819	reg->type |= NON_OWN_REF;
12820	if (rec->refcount_off >= 0)
12821		reg->type |= MEM_RCU;
12822
12823	return 0;
12824}
12825
12826static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id)
12827{
12828	struct bpf_verifier_state *state = env->cur_state;
12829	struct bpf_func_state *unused;
12830	struct bpf_reg_state *reg;
12831	int i;
12832
12833	if (!ref_obj_id) {
12834		verifier_bug(env, "ref_obj_id is zero for owning -> non-owning conversion");
12835		return -EFAULT;
12836	}
12837
12838	for (i = 0; i < state->acquired_refs; i++) {
12839		if (state->refs[i].id != ref_obj_id)
12840			continue;
12841
12842		/* Clear ref_obj_id here so release_reference doesn't clobber
12843		 * the whole reg
12844		 */
12845		bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({
12846			if (reg->ref_obj_id == ref_obj_id) {
12847				reg->ref_obj_id = 0;
12848				ref_set_non_owning(env, reg);
12849			}
12850		}));
12851		return 0;
12852	}
12853
12854	verifier_bug(env, "ref state missing for ref_obj_id");
12855	return -EFAULT;
12856}
12857
12858/* Implementation details:
12859 *
12860 * Each register points to some region of memory, which we define as an
12861 * allocation. Each allocation may embed a bpf_spin_lock which protects any
12862 * special BPF objects (bpf_list_head, bpf_rb_root, etc.) part of the same
12863 * allocation. The lock and the data it protects are colocated in the same
12864 * memory region.
12865 *
12866 * Hence, everytime a register holds a pointer value pointing to such
12867 * allocation, the verifier preserves a unique reg->id for it.
12868 *
12869 * The verifier remembers the lock 'ptr' and the lock 'id' whenever
12870 * bpf_spin_lock is called.
12871 *
12872 * To enable this, lock state in the verifier captures two values:
12873 *	active_lock.ptr = Register's type specific pointer
12874 *	active_lock.id  = A unique ID for each register pointer value
12875 *
12876 * Currently, PTR_TO_MAP_VALUE and PTR_TO_BTF_ID | MEM_ALLOC are the two
12877 * supported register types.
12878 *
12879 * The active_lock.ptr in case of map values is the reg->map_ptr, and in case of
12880 * allocated objects is the reg->btf pointer.
12881 *
12882 * The active_lock.id is non-unique for maps supporting direct_value_addr, as we
12883 * can establish the provenance of the map value statically for each distinct
12884 * lookup into such maps. They always contain a single map value hence unique
12885 * IDs for each pseudo load pessimizes the algorithm and rejects valid programs.
12886 *
12887 * So, in case of global variables, they use array maps with max_entries = 1,
12888 * hence their active_lock.ptr becomes map_ptr and id = 0 (since they all point
12889 * into the same map value as max_entries is 1, as described above).
12890 *
12891 * In case of inner map lookups, the inner map pointer has same map_ptr as the
12892 * outer map pointer (in verifier context), but each lookup into an inner map
12893 * assigns a fresh reg->id to the lookup, so while lookups into distinct inner
12894 * maps from the same outer map share the same map_ptr as active_lock.ptr, they
12895 * will get different reg->id assigned to each lookup, hence different
12896 * active_lock.id.
12897 *
12898 * In case of allocated objects, active_lock.ptr is the reg->btf, and the
12899 * reg->id is a unique ID preserved after the NULL pointer check on the pointer
12900 * returned from bpf_obj_new. Each allocation receives a new reg->id.
12901 */
12902static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
12903{
12904	struct bpf_reference_state *s;
12905	void *ptr;
12906	u32 id;
12907
12908	switch ((int)reg->type) {
12909	case PTR_TO_MAP_VALUE:
12910		ptr = reg->map_ptr;
12911		break;
12912	case PTR_TO_BTF_ID | MEM_ALLOC:
12913		ptr = reg->btf;
12914		break;
12915	default:
12916		verifier_bug(env, "unknown reg type for lock check");
12917		return -EFAULT;
12918	}
12919	id = reg->id;
12920
12921	if (!env->cur_state->active_locks)
12922		return -EINVAL;
12923	s = find_lock_state(env->cur_state, REF_TYPE_LOCK_MASK, id, ptr);
12924	if (!s) {
12925		verbose(env, "held lock and object are not in the same allocation\n");
12926		return -EINVAL;
12927	}
12928	return 0;
12929}
12930
12931static bool is_bpf_list_api_kfunc(u32 btf_id)
12932{
12933	return btf_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
12934	       btf_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||
12935	       btf_id == special_kfunc_list[KF_bpf_list_pop_front] ||
12936	       btf_id == special_kfunc_list[KF_bpf_list_pop_back] ||
12937	       btf_id == special_kfunc_list[KF_bpf_list_front] ||
12938	       btf_id == special_kfunc_list[KF_bpf_list_back];
12939}
12940
12941static bool is_bpf_rbtree_api_kfunc(u32 btf_id)
12942{
12943	return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl] ||
12944	       btf_id == special_kfunc_list[KF_bpf_rbtree_remove] ||
12945	       btf_id == special_kfunc_list[KF_bpf_rbtree_first] ||
12946	       btf_id == special_kfunc_list[KF_bpf_rbtree_root] ||
12947	       btf_id == special_kfunc_list[KF_bpf_rbtree_left] ||
12948	       btf_id == special_kfunc_list[KF_bpf_rbtree_right];
12949}
12950
12951static bool is_bpf_iter_num_api_kfunc(u32 btf_id)
12952{
12953	return btf_id == special_kfunc_list[KF_bpf_iter_num_new] ||
12954	       btf_id == special_kfunc_list[KF_bpf_iter_num_next] ||
12955	       btf_id == special_kfunc_list[KF_bpf_iter_num_destroy];
12956}
12957
12958static bool is_bpf_graph_api_kfunc(u32 btf_id)
12959{
12960	return is_bpf_list_api_kfunc(btf_id) || is_bpf_rbtree_api_kfunc(btf_id) ||
12961	       btf_id == special_kfunc_list[KF_bpf_refcount_acquire_impl];
12962}
12963
12964static bool is_bpf_res_spin_lock_kfunc(u32 btf_id)
12965{
12966	return btf_id == special_kfunc_list[KF_bpf_res_spin_lock] ||
12967	       btf_id == special_kfunc_list[KF_bpf_res_spin_unlock] ||
12968	       btf_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] ||
12969	       btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore];
12970}
12971
12972static bool kfunc_spin_allowed(u32 btf_id)
12973{
12974	return is_bpf_graph_api_kfunc(btf_id) || is_bpf_iter_num_api_kfunc(btf_id) ||
12975	       is_bpf_res_spin_lock_kfunc(btf_id);
12976}
12977
12978static bool is_sync_callback_calling_kfunc(u32 btf_id)
12979{
12980	return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl];
12981}
12982
12983static bool is_async_callback_calling_kfunc(u32 btf_id)
12984{
12985	return btf_id == special_kfunc_list[KF_bpf_wq_set_callback_impl] ||
12986	       is_task_work_add_kfunc(btf_id);
12987}
12988
12989static bool is_bpf_throw_kfunc(struct bpf_insn *insn)
12990{
12991	return bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
12992	       insn->imm == special_kfunc_list[KF_bpf_throw];
12993}
12994
12995static bool is_bpf_wq_set_callback_impl_kfunc(u32 btf_id)
12996{
12997	return btf_id == special_kfunc_list[KF_bpf_wq_set_callback_impl];
12998}
12999
13000static bool is_callback_calling_kfunc(u32 btf_id)
13001{
13002	return is_sync_callback_calling_kfunc(btf_id) ||
13003	       is_async_callback_calling_kfunc(btf_id);
13004}
13005
13006static bool is_rbtree_lock_required_kfunc(u32 btf_id)
13007{
13008	return is_bpf_rbtree_api_kfunc(btf_id);
13009}
13010
13011static bool check_kfunc_is_graph_root_api(struct bpf_verifier_env *env,
13012					  enum btf_field_type head_field_type,
13013					  u32 kfunc_btf_id)
13014{
13015	bool ret;
13016
13017	switch (head_field_type) {
13018	case BPF_LIST_HEAD:
13019		ret = is_bpf_list_api_kfunc(kfunc_btf_id);
13020		break;
13021	case BPF_RB_ROOT:
13022		ret = is_bpf_rbtree_api_kfunc(kfunc_btf_id);
13023		break;
13024	default:
13025		verbose(env, "verifier internal error: unexpected graph root argument type %s\n",
13026			btf_field_type_name(head_field_type));
13027		return false;
13028	}
13029
13030	if (!ret)
13031		verbose(env, "verifier internal error: %s head arg for unknown kfunc\n",
13032			btf_field_type_name(head_field_type));
13033	return ret;
13034}
13035
13036static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env,
13037					  enum btf_field_type node_field_type,
13038					  u32 kfunc_btf_id)
13039{
13040	bool ret;
13041
13042	switch (node_field_type) {
13043	case BPF_LIST_NODE:
13044		ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
13045		       kfunc_btf_id == special_kfunc_list[KF_bpf_list_push_back_impl]);
13046		break;
13047	case BPF_RB_NODE:
13048		ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_remove] ||
13049		       kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl] ||
13050		       kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_left] ||
13051		       kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_right]);
13052		break;
13053	default:
13054		verbose(env, "verifier internal error: unexpected graph node argument type %s\n",
13055			btf_field_type_name(node_field_type));
13056		return false;
13057	}
13058
13059	if (!ret)
13060		verbose(env, "verifier internal error: %s node arg for unknown kfunc\n",
13061			btf_field_type_name(node_field_type));
13062	return ret;
13063}
13064
13065static int
13066__process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env,
13067				   struct bpf_reg_state *reg, u32 regno,
13068				   struct bpf_kfunc_call_arg_meta *meta,
13069				   enum btf_field_type head_field_type,
13070				   struct btf_field **head_field)
13071{
13072	const char *head_type_name;
13073	struct btf_field *field;
13074	struct btf_record *rec;
13075	u32 head_off;
13076
13077	if (meta->btf != btf_vmlinux) {
13078		verifier_bug(env, "unexpected btf mismatch in kfunc call");
13079		return -EFAULT;
13080	}
13081
13082	if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id))
13083		return -EFAULT;
13084
13085	head_type_name = btf_field_type_name(head_field_type);
13086	if (!tnum_is_const(reg->var_off)) {
13087		verbose(env,
13088			"R%d doesn't have constant offset. %s has to be at the constant offset\n",
13089			regno, head_type_name);
13090		return -EINVAL;
13091	}
13092
13093	rec = reg_btf_record(reg);
13094	head_off = reg->off + reg->var_off.value;
13095	field = btf_record_find(rec, head_off, head_field_type);
13096	if (!field) {
13097		verbose(env, "%s not found at offset=%u\n", head_type_name, head_off);
13098		return -EINVAL;
13099	}
13100
13101	/* All functions require bpf_list_head to be protected using a bpf_spin_lock */
13102	if (check_reg_allocation_locked(env, reg)) {
13103		verbose(env, "bpf_spin_lock at off=%d must be held for %s\n",
13104			rec->spin_lock_off, head_type_name);
13105		return -EINVAL;
13106	}
13107
13108	if (*head_field) {
13109		verifier_bug(env, "repeating %s arg", head_type_name);
13110		return -EFAULT;
13111	}
13112	*head_field = field;
13113	return 0;
13114}
13115
13116static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env,
13117					   struct bpf_reg_state *reg, u32 regno,
13118					   struct bpf_kfunc_call_arg_meta *meta)
13119{
13120	return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_LIST_HEAD,
13121							  &meta->arg_list_head.field);
13122}
13123
13124static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env,
13125					     struct bpf_reg_state *reg, u32 regno,
13126					     struct bpf_kfunc_call_arg_meta *meta)
13127{
13128	return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_RB_ROOT,
13129							  &meta->arg_rbtree_root.field);
13130}
13131
13132static int
13133__process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,
13134				   struct bpf_reg_state *reg, u32 regno,
13135				   struct bpf_kfunc_call_arg_meta *meta,
13136				   enum btf_field_type head_field_type,
13137				   enum btf_field_type node_field_type,
13138				   struct btf_field **node_field)
13139{
13140	const char *node_type_name;
13141	const struct btf_type *et, *t;
13142	struct btf_field *field;
13143	u32 node_off;
13144
13145	if (meta->btf != btf_vmlinux) {
13146		verifier_bug(env, "unexpected btf mismatch in kfunc call");
13147		return -EFAULT;
13148	}
13149
13150	if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id))
13151		return -EFAULT;
13152
13153	node_type_name = btf_field_type_name(node_field_type);
13154	if (!tnum_is_const(reg->var_off)) {
13155		verbose(env,
13156			"R%d doesn't have constant offset. %s has to be at the constant offset\n",
13157			regno, node_type_name);
13158		return -EINVAL;
13159	}
13160
13161	node_off = reg->off + reg->var_off.value;
13162	field = reg_find_field_offset(reg, node_off, node_field_type);
13163	if (!field) {
13164		verbose(env, "%s not found at offset=%u\n", node_type_name, node_off);
13165		return -EINVAL;
13166	}
13167
13168	field = *node_field;
13169
13170	et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id);
13171	t = btf_type_by_id(reg->btf, reg->btf_id);
13172	if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf,
13173				  field->graph_root.value_btf_id, true)) {
13174		verbose(env, "operation on %s expects arg#1 %s at offset=%d "
13175			"in struct %s, but arg is at offset=%d in struct %s\n",
13176			btf_field_type_name(head_field_type),
13177			btf_field_type_name(node_field_type),
13178			field->graph_root.node_offset,
13179			btf_name_by_offset(field->graph_root.btf, et->name_off),
13180			node_off, btf_name_by_offset(reg->btf, t->name_off));
13181		return -EINVAL;
13182	}
13183	meta->arg_btf = reg->btf;
13184	meta->arg_btf_id = reg->btf_id;
13185
13186	if (node_off != field->graph_root.node_offset) {
13187		verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n",
13188			node_off, btf_field_type_name(node_field_type),
13189			field->graph_root.node_offset,
13190			btf_name_by_offset(field->graph_root.btf, et->name_off));
13191		return -EINVAL;
13192	}
13193
13194	return 0;
13195}
13196
13197static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env,
13198					   struct bpf_reg_state *reg, u32 regno,
13199					   struct bpf_kfunc_call_arg_meta *meta)
13200{
13201	return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta,
13202						  BPF_LIST_HEAD, BPF_LIST_NODE,
13203						  &meta->arg_list_head.field);
13204}
13205
13206static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env,
13207					     struct bpf_reg_state *reg, u32 regno,
13208					     struct bpf_kfunc_call_arg_meta *meta)
13209{
13210	return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta,
13211						  BPF_RB_ROOT, BPF_RB_NODE,
13212						  &meta->arg_rbtree_root.field);
13213}
13214
13215/*
13216 * css_task iter allowlist is needed to avoid dead locking on css_set_lock.
13217 * LSM hooks and iters (both sleepable and non-sleepable) are safe.
13218 * Any sleepable progs are also safe since bpf_check_attach_target() enforce
13219 * them can only be attached to some specific hook points.
13220 */
13221static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env)
13222{
13223	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
13224
13225	switch (prog_type) {
13226	case BPF_PROG_TYPE_LSM:
13227		return true;
13228	case BPF_PROG_TYPE_TRACING:
13229		if (env->prog->expected_attach_type == BPF_TRACE_ITER)
13230			return true;
13231		fallthrough;
13232	default:
13233		return in_sleepable(env);
13234	}
13235}
13236
13237static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta,
13238			    int insn_idx)
13239{
13240	const char *func_name = meta->func_name, *ref_tname;
13241	const struct btf *btf = meta->btf;
13242	const struct btf_param *args;
13243	struct btf_record *rec;
13244	u32 i, nargs;
13245	int ret;
13246
13247	args = (const struct btf_param *)(meta->func_proto + 1);
13248	nargs = btf_type_vlen(meta->func_proto);
13249	if (nargs > MAX_BPF_FUNC_REG_ARGS) {
13250		verbose(env, "Function %s has %d > %d args\n", func_name, nargs,
13251			MAX_BPF_FUNC_REG_ARGS);
13252		return -EINVAL;
13253	}
13254
13255	/* Check that BTF function arguments match actual types that the
13256	 * verifier sees.
13257	 */
13258	for (i = 0; i < nargs; i++) {
13259		struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[i + 1];
13260		const struct btf_type *t, *ref_t, *resolve_ret;
13261		enum bpf_arg_type arg_type = ARG_DONTCARE;
13262		u32 regno = i + 1, ref_id, type_size;
13263		bool is_ret_buf_sz = false;
13264		int kf_arg_type;
13265
13266		t = btf_type_skip_modifiers(btf, args[i].type, NULL);
13267
13268		if (is_kfunc_arg_ignore(btf, &args[i]))
13269			continue;
13270
13271		if (is_kfunc_arg_prog(btf, &args[i])) {
13272			/* Used to reject repeated use of __prog. */
13273			if (meta->arg_prog) {
13274				verifier_bug(env, "Only 1 prog->aux argument supported per-kfunc");
13275				return -EFAULT;
13276			}
13277			meta->arg_prog = true;
13278			cur_aux(env)->arg_prog = regno;
13279			continue;
13280		}
13281
13282		if (btf_type_is_scalar(t)) {
13283			if (reg->type != SCALAR_VALUE) {
13284				verbose(env, "R%d is not a scalar\n", regno);
13285				return -EINVAL;
13286			}
13287
13288			if (is_kfunc_arg_constant(meta->btf, &args[i])) {
13289				if (meta->arg_constant.found) {
13290					verifier_bug(env, "only one constant argument permitted");
13291					return -EFAULT;
13292				}
13293				if (!tnum_is_const(reg->var_off)) {
13294					verbose(env, "R%d must be a known constant\n", regno);
13295					return -EINVAL;
13296				}
13297				ret = mark_chain_precision(env, regno);
13298				if (ret < 0)
13299					return ret;
13300				meta->arg_constant.found = true;
13301				meta->arg_constant.value = reg->var_off.value;
13302			} else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) {
13303				meta->r0_rdonly = true;
13304				is_ret_buf_sz = true;
13305			} else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdwr_buf_size")) {
13306				is_ret_buf_sz = true;
13307			}
13308
13309			if (is_ret_buf_sz) {
13310				if (meta->r0_size) {
13311					verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc");
13312					return -EINVAL;
13313				}
13314
13315				if (!tnum_is_const(reg->var_off)) {
13316					verbose(env, "R%d is not a const\n", regno);
13317					return -EINVAL;
13318				}
13319
13320				meta->r0_size = reg->var_off.value;
13321				ret = mark_chain_precision(env, regno);
13322				if (ret)
13323					return ret;
13324			}
13325			continue;
13326		}
13327
13328		if (!btf_type_is_ptr(t)) {
13329			verbose(env, "Unrecognized arg#%d type %s\n", i, btf_type_str(t));
13330			return -EINVAL;
13331		}
13332
13333		if ((is_kfunc_trusted_args(meta) || is_kfunc_rcu(meta)) &&
13334		    (register_is_null(reg) || type_may_be_null(reg->type)) &&
13335			!is_kfunc_arg_nullable(meta->btf, &args[i])) {
13336			verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i);
13337			return -EACCES;
13338		}
13339
13340		if (reg->ref_obj_id) {
13341			if (is_kfunc_release(meta) && meta->ref_obj_id) {
13342				verifier_bug(env, "more than one arg with ref_obj_id R%d %u %u",
13343					     regno, reg->ref_obj_id,
13344					     meta->ref_obj_id);
13345				return -EFAULT;
13346			}
13347			meta->ref_obj_id = reg->ref_obj_id;
13348			if (is_kfunc_release(meta))
13349				meta->release_regno = regno;
13350		}
13351
13352		ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
13353		ref_tname = btf_name_by_offset(btf, ref_t->name_off);
13354
13355		kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs);
13356		if (kf_arg_type < 0)
13357			return kf_arg_type;
13358
13359		switch (kf_arg_type) {
13360		case KF_ARG_PTR_TO_NULL:
13361			continue;
13362		case KF_ARG_PTR_TO_MAP:
13363			if (!reg->map_ptr) {
13364				verbose(env, "pointer in R%d isn't map pointer\n", regno);
13365				return -EINVAL;
13366			}
13367			if (meta->map.ptr && (reg->map_ptr->record->wq_off >= 0 ||
13368					      reg->map_ptr->record->task_work_off >= 0)) {
13369				/* Use map_uid (which is unique id of inner map) to reject:
13370				 * inner_map1 = bpf_map_lookup_elem(outer_map, key1)
13371				 * inner_map2 = bpf_map_lookup_elem(outer_map, key2)
13372				 * if (inner_map1 && inner_map2) {
13373				 *     wq = bpf_map_lookup_elem(inner_map1);
13374				 *     if (wq)
13375				 *         // mismatch would have been allowed
13376				 *         bpf_wq_init(wq, inner_map2);
13377				 * }
13378				 *
13379				 * Comparing map_ptr is enough to distinguish normal and outer maps.
13380				 */
13381				if (meta->map.ptr != reg->map_ptr ||
13382				    meta->map.uid != reg->map_uid) {
13383					if (reg->map_ptr->record->task_work_off >= 0) {
13384						verbose(env,
13385							"bpf_task_work pointer in R2 map_uid=%d doesn't match map pointer in R3 map_uid=%d\n",
13386							meta->map.uid, reg->map_uid);
13387						return -EINVAL;
13388					}
13389					verbose(env,
13390						"workqueue pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n",
13391						meta->map.uid, reg->map_uid);
13392					return -EINVAL;
13393				}
13394			}
13395			meta->map.ptr = reg->map_ptr;
13396			meta->map.uid = reg->map_uid;
13397			fallthrough;
13398		case KF_ARG_PTR_TO_ALLOC_BTF_ID:
13399		case KF_ARG_PTR_TO_BTF_ID:
13400			if (!is_kfunc_trusted_args(meta) && !is_kfunc_rcu(meta))
13401				break;
13402
13403			if (!is_trusted_reg(reg)) {
13404				if (!is_kfunc_rcu(meta)) {
13405					verbose(env, "R%d must be referenced or trusted\n", regno);
13406					return -EINVAL;
13407				}
13408				if (!is_rcu_reg(reg)) {
13409					verbose(env, "R%d must be a rcu pointer\n", regno);
13410					return -EINVAL;
13411				}
13412			}
13413			fallthrough;
13414		case KF_ARG_PTR_TO_CTX:
13415		case KF_ARG_PTR_TO_DYNPTR:
13416		case KF_ARG_PTR_TO_ITER:
13417		case KF_ARG_PTR_TO_LIST_HEAD:
13418		case KF_ARG_PTR_TO_LIST_NODE:
13419		case KF_ARG_PTR_TO_RB_ROOT:
13420		case KF_ARG_PTR_TO_RB_NODE:
13421		case KF_ARG_PTR_TO_MEM:
13422		case KF_ARG_PTR_TO_MEM_SIZE:
13423		case KF_ARG_PTR_TO_CALLBACK:
13424		case KF_ARG_PTR_TO_REFCOUNTED_KPTR:
13425		case KF_ARG_PTR_TO_CONST_STR:
13426		case KF_ARG_PTR_TO_WORKQUEUE:
13427		case KF_ARG_PTR_TO_TASK_WORK:
13428		case KF_ARG_PTR_TO_IRQ_FLAG:
13429		case KF_ARG_PTR_TO_RES_SPIN_LOCK:
13430			break;
13431		default:
13432			verifier_bug(env, "unknown kfunc arg type %d", kf_arg_type);
13433			return -EFAULT;
13434		}
13435
13436		if (is_kfunc_release(meta) && reg->ref_obj_id)
13437			arg_type |= OBJ_RELEASE;
13438		ret = check_func_arg_reg_off(env, reg, regno, arg_type);
13439		if (ret < 0)
13440			return ret;
13441
13442		switch (kf_arg_type) {
13443		case KF_ARG_PTR_TO_CTX:
13444			if (reg->type != PTR_TO_CTX) {
13445				verbose(env, "arg#%d expected pointer to ctx, but got %s\n",
13446					i, reg_type_str(env, reg->type));
13447				return -EINVAL;
13448			}
13449
13450			if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) {
13451				ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog));
13452				if (ret < 0)
13453					return -EINVAL;
13454				meta->ret_btf_id  = ret;
13455			}
13456			break;
13457		case KF_ARG_PTR_TO_ALLOC_BTF_ID:
13458			if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) {
13459				if (meta->func_id != special_kfunc_list[KF_bpf_obj_drop_impl]) {
13460					verbose(env, "arg#%d expected for bpf_obj_drop_impl()\n", i);
13461					return -EINVAL;
13462				}
13463			} else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) {
13464				if (meta->func_id != special_kfunc_list[KF_bpf_percpu_obj_drop_impl]) {
13465					verbose(env, "arg#%d expected for bpf_percpu_obj_drop_impl()\n", i);
13466					return -EINVAL;
13467				}
13468			} else {
13469				verbose(env, "arg#%d expected pointer to allocated object\n", i);
13470				return -EINVAL;
13471			}
13472			if (!reg->ref_obj_id) {
13473				verbose(env, "allocated object must be referenced\n");
13474				return -EINVAL;
13475			}
13476			if (meta->btf == btf_vmlinux) {
13477				meta->arg_btf = reg->btf;
13478				meta->arg_btf_id = reg->btf_id;
13479			}
13480			break;
13481		case KF_ARG_PTR_TO_DYNPTR:
13482		{
13483			enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
13484			int clone_ref_obj_id = 0;
13485
13486			if (reg->type == CONST_PTR_TO_DYNPTR)
13487				dynptr_arg_type |= MEM_RDONLY;
13488
13489			if (is_kfunc_arg_uninit(btf, &args[i]))
13490				dynptr_arg_type |= MEM_UNINIT;
13491
13492			if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
13493				dynptr_arg_type |= DYNPTR_TYPE_SKB;
13494			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp]) {
13495				dynptr_arg_type |= DYNPTR_TYPE_XDP;
13496			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb_meta]) {
13497				dynptr_arg_type |= DYNPTR_TYPE_SKB_META;
13498			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_file]) {
13499				dynptr_arg_type |= DYNPTR_TYPE_FILE;
13500			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_file_discard]) {
13501				dynptr_arg_type |= DYNPTR_TYPE_FILE;
13502				meta->release_regno = regno;
13503			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] &&
13504				   (dynptr_arg_type & MEM_UNINIT)) {
13505				enum bpf_dynptr_type parent_type = meta->initialized_dynptr.type;
13506
13507				if (parent_type == BPF_DYNPTR_TYPE_INVALID) {
13508					verifier_bug(env, "no dynptr type for parent of clone");
13509					return -EFAULT;
13510				}
13511
13512				dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type);
13513				clone_ref_obj_id = meta->initialized_dynptr.ref_obj_id;
13514				if (dynptr_type_refcounted(parent_type) && !clone_ref_obj_id) {
13515					verifier_bug(env, "missing ref obj id for parent of clone");
13516					return -EFAULT;
13517				}
13518			}
13519
13520			ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type, clone_ref_obj_id);
13521			if (ret < 0)
13522				return ret;
13523
13524			if (!(dynptr_arg_type & MEM_UNINIT)) {
13525				int id = dynptr_id(env, reg);
13526
13527				if (id < 0) {
13528					verifier_bug(env, "failed to obtain dynptr id");
13529					return id;
13530				}
13531				meta->initialized_dynptr.id = id;
13532				meta->initialized_dynptr.type = dynptr_get_type(env, reg);
13533				meta->initialized_dynptr.ref_obj_id = dynptr_ref_obj_id(env, reg);
13534			}
13535
13536			break;
13537		}
13538		case KF_ARG_PTR_TO_ITER:
13539			if (meta->func_id == special_kfunc_list[KF_bpf_iter_css_task_new]) {
13540				if (!check_css_task_iter_allowlist(env)) {
13541					verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n");
13542					return -EINVAL;
13543				}
13544			}
13545			ret = process_iter_arg(env, regno, insn_idx, meta);
13546			if (ret < 0)
13547				return ret;
13548			break;
13549		case KF_ARG_PTR_TO_LIST_HEAD:
13550			if (reg->type != PTR_TO_MAP_VALUE &&
13551			    reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
13552				verbose(env, "arg#%d expected pointer to map value or allocated object\n", i);
13553				return -EINVAL;
13554			}
13555			if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) {
13556				verbose(env, "allocated object must be referenced\n");
13557				return -EINVAL;
13558			}
13559			ret = process_kf_arg_ptr_to_list_head(env, reg, regno, meta);
13560			if (ret < 0)
13561				return ret;
13562			break;
13563		case KF_ARG_PTR_TO_RB_ROOT:
13564			if (reg->type != PTR_TO_MAP_VALUE &&
13565			    reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
13566				verbose(env, "arg#%d expected pointer to map value or allocated object\n", i);
13567				return -EINVAL;
13568			}
13569			if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) {
13570				verbose(env, "allocated object must be referenced\n");
13571				return -EINVAL;
13572			}
13573			ret = process_kf_arg_ptr_to_rbtree_root(env, reg, regno, meta);
13574			if (ret < 0)
13575				return ret;
13576			break;
13577		case KF_ARG_PTR_TO_LIST_NODE:
13578			if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
13579				verbose(env, "arg#%d expected pointer to allocated object\n", i);
13580				return -EINVAL;
13581			}
13582			if (!reg->ref_obj_id) {
13583				verbose(env, "allocated object must be referenced\n");
13584				return -EINVAL;
13585			}
13586			ret = process_kf_arg_ptr_to_list_node(env, reg, regno, meta);
13587			if (ret < 0)
13588				return ret;
13589			break;
13590		case KF_ARG_PTR_TO_RB_NODE:
13591			if (meta->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
13592				if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
13593					verbose(env, "arg#%d expected pointer to allocated object\n", i);
13594					return -EINVAL;
13595				}
13596				if (!reg->ref_obj_id) {
13597					verbose(env, "allocated object must be referenced\n");
13598					return -EINVAL;
13599				}
13600			} else {
13601				if (!type_is_non_owning_ref(reg->type) && !reg->ref_obj_id) {
13602					verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", func_name);
13603					return -EINVAL;
13604				}
13605				if (in_rbtree_lock_required_cb(env)) {
13606					verbose(env, "%s not allowed in rbtree cb\n", func_name);
13607					return -EINVAL;
13608				}
13609			}
13610
13611			ret = process_kf_arg_ptr_to_rbtree_node(env, reg, regno, meta);
13612			if (ret < 0)
13613				return ret;
13614			break;
13615		case KF_ARG_PTR_TO_MAP:
13616			/* If argument has '__map' suffix expect 'struct bpf_map *' */
13617			ref_id = *reg2btf_ids[CONST_PTR_TO_MAP];
13618			ref_t = btf_type_by_id(btf_vmlinux, ref_id);
13619			ref_tname = btf_name_by_offset(btf, ref_t->name_off);
13620			fallthrough;
13621		case KF_ARG_PTR_TO_BTF_ID:
13622			/* Only base_type is checked, further checks are done here */
13623			if ((base_type(reg->type) != PTR_TO_BTF_ID ||
13624			     (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) &&
13625			    !reg2btf_ids[base_type(reg->type)]) {
13626				verbose(env, "arg#%d is %s ", i, reg_type_str(env, reg->type));
13627				verbose(env, "expected %s or socket\n",
13628					reg_type_str(env, base_type(reg->type) |
13629							  (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS)));
13630				return -EINVAL;
13631			}
13632			ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i);
13633			if (ret < 0)
13634				return ret;
13635			break;
13636		case KF_ARG_PTR_TO_MEM:
13637			resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
13638			if (IS_ERR(resolve_ret)) {
13639				verbose(env, "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
13640					i, btf_type_str(ref_t), ref_tname, PTR_ERR(resolve_ret));
13641				return -EINVAL;
13642			}
13643			ret = check_mem_reg(env, reg, regno, type_size);
13644			if (ret < 0)
13645				return ret;
13646			break;
13647		case KF_ARG_PTR_TO_MEM_SIZE:
13648		{
13649			struct bpf_reg_state *buff_reg = &regs[regno];
13650			const struct btf_param *buff_arg = &args[i];
13651			struct bpf_reg_state *size_reg = &regs[regno + 1];
13652			const struct btf_param *size_arg = &args[i + 1];
13653
13654			if (!register_is_null(buff_reg) || !is_kfunc_arg_optional(meta->btf, buff_arg)) {
13655				ret = check_kfunc_mem_size_reg(env, size_reg, regno + 1);
13656				if (ret < 0) {
13657					verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1);
13658					return ret;
13659				}
13660			}
13661
13662			if (is_kfunc_arg_const_mem_size(meta->btf, size_arg, size_reg)) {
13663				if (meta->arg_constant.found) {
13664					verifier_bug(env, "only one constant argument permitted");
13665					return -EFAULT;
13666				}
13667				if (!tnum_is_const(size_reg->var_off)) {
13668					verbose(env, "R%d must be a known constant\n", regno + 1);
13669					return -EINVAL;
13670				}
13671				meta->arg_constant.found = true;
13672				meta->arg_constant.value = size_reg->var_off.value;
13673			}
13674
13675			/* Skip next '__sz' or '__szk' argument */
13676			i++;
13677			break;
13678		}
13679		case KF_ARG_PTR_TO_CALLBACK:
13680			if (reg->type != PTR_TO_FUNC) {
13681				verbose(env, "arg%d expected pointer to func\n", i);
13682				return -EINVAL;
13683			}
13684			meta->subprogno = reg->subprogno;
13685			break;
13686		case KF_ARG_PTR_TO_REFCOUNTED_KPTR:
13687			if (!type_is_ptr_alloc_obj(reg->type)) {
13688				verbose(env, "arg#%d is neither owning or non-owning ref\n", i);
13689				return -EINVAL;
13690			}
13691			if (!type_is_non_owning_ref(reg->type))
13692				meta->arg_owning_ref = true;
13693
13694			rec = reg_btf_record(reg);
13695			if (!rec) {
13696				verifier_bug(env, "Couldn't find btf_record");
13697				return -EFAULT;
13698			}
13699
13700			if (rec->refcount_off < 0) {
13701				verbose(env, "arg#%d doesn't point to a type with bpf_refcount field\n", i);
13702				return -EINVAL;
13703			}
13704
13705			meta->arg_btf = reg->btf;
13706			meta->arg_btf_id = reg->btf_id;
13707			break;
13708		case KF_ARG_PTR_TO_CONST_STR:
13709			if (reg->type != PTR_TO_MAP_VALUE) {
13710				verbose(env, "arg#%d doesn't point to a const string\n", i);
13711				return -EINVAL;
13712			}
13713			ret = check_reg_const_str(env, reg, regno);
13714			if (ret)
13715				return ret;
13716			break;
13717		case KF_ARG_PTR_TO_WORKQUEUE:
13718			if (reg->type != PTR_TO_MAP_VALUE) {
13719				verbose(env, "arg#%d doesn't point to a map value\n", i);
13720				return -EINVAL;
13721			}
13722			ret = process_wq_func(env, regno, meta);
13723			if (ret < 0)
13724				return ret;
13725			break;
13726		case KF_ARG_PTR_TO_TASK_WORK:
13727			if (reg->type != PTR_TO_MAP_VALUE) {
13728				verbose(env, "arg#%d doesn't point to a map value\n", i);
13729				return -EINVAL;
13730			}
13731			ret = process_task_work_func(env, regno, meta);
13732			if (ret < 0)
13733				return ret;
13734			break;
13735		case KF_ARG_PTR_TO_IRQ_FLAG:
13736			if (reg->type != PTR_TO_STACK) {
13737				verbose(env, "arg#%d doesn't point to an irq flag on stack\n", i);
13738				return -EINVAL;
13739			}
13740			ret = process_irq_flag(env, regno, meta);
13741			if (ret < 0)
13742				return ret;
13743			break;
13744		case KF_ARG_PTR_TO_RES_SPIN_LOCK:
13745		{
13746			int flags = PROCESS_RES_LOCK;
13747
13748			if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
13749				verbose(env, "arg#%d doesn't point to map value or allocated object\n", i);
13750				return -EINVAL;
13751			}
13752
13753			if (!is_bpf_res_spin_lock_kfunc(meta->func_id))
13754				return -EFAULT;
13755			if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock] ||
13756			    meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])
13757				flags |= PROCESS_SPIN_LOCK;
13758			if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] ||
13759			    meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])
13760				flags |= PROCESS_LOCK_IRQ;
13761			ret = process_spin_lock(env, regno, flags);
13762			if (ret < 0)
13763				return ret;
13764			break;
13765		}
13766		}
13767	}
13768
13769	if (is_kfunc_release(meta) && !meta->release_regno) {
13770		verbose(env, "release kernel function %s expects refcounted PTR_TO_BTF_ID\n",
13771			func_name);
13772		return -EINVAL;
13773	}
13774
13775	return 0;
13776}
13777
13778static int fetch_kfunc_meta(struct bpf_verifier_env *env,
13779			    struct bpf_insn *insn,
13780			    struct bpf_kfunc_call_arg_meta *meta,
13781			    const char **kfunc_name)
13782{
13783	const struct btf_type *func, *func_proto;
13784	u32 func_id, *kfunc_flags;
13785	const char *func_name;
13786	struct btf *desc_btf;
13787
13788	if (kfunc_name)
13789		*kfunc_name = NULL;
13790
13791	if (!insn->imm)
13792		return -EINVAL;
13793
13794	desc_btf = find_kfunc_desc_btf(env, insn->off);
13795	if (IS_ERR(desc_btf))
13796		return PTR_ERR(desc_btf);
13797
13798	func_id = insn->imm;
13799	func = btf_type_by_id(desc_btf, func_id);
13800	func_name = btf_name_by_offset(desc_btf, func->name_off);
13801	if (kfunc_name)
13802		*kfunc_name = func_name;
13803	func_proto = btf_type_by_id(desc_btf, func->type);
13804
13805	kfunc_flags = btf_kfunc_id_set_contains(desc_btf, func_id, env->prog);
13806	if (!kfunc_flags) {
13807		return -EACCES;
13808	}
13809
13810	memset(meta, 0, sizeof(*meta));
13811	meta->btf = desc_btf;
13812	meta->func_id = func_id;
13813	meta->kfunc_flags = *kfunc_flags;
13814	meta->func_proto = func_proto;
13815	meta->func_name = func_name;
13816
13817	return 0;
13818}
13819
13820/* check special kfuncs and return:
13821 *  1  - not fall-through to 'else' branch, continue verification
13822 *  0  - fall-through to 'else' branch
13823 * < 0 - not fall-through to 'else' branch, return error
13824 */
13825static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta,
13826			       struct bpf_reg_state *regs, struct bpf_insn_aux_data *insn_aux,
13827			       const struct btf_type *ptr_type, struct btf *desc_btf)
13828{
13829	const struct btf_type *ret_t;
13830	int err = 0;
13831
13832	if (meta->btf != btf_vmlinux)
13833		return 0;
13834
13835	if (meta->func_id == special_kfunc_list[KF_bpf_obj_new_impl] ||
13836	    meta->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
13837		struct btf_struct_meta *struct_meta;
13838		struct btf *ret_btf;
13839		u32 ret_btf_id;
13840
13841		if (meta->func_id == special_kfunc_list[KF_bpf_obj_new_impl] && !bpf_global_ma_set)
13842			return -ENOMEM;
13843
13844		if (((u64)(u32)meta->arg_constant.value) != meta->arg_constant.value) {
13845			verbose(env, "local type ID argument must be in range [0, U32_MAX]\n");
13846			return -EINVAL;
13847		}
13848
13849		ret_btf = env->prog->aux->btf;
13850		ret_btf_id = meta->arg_constant.value;
13851
13852		/* This may be NULL due to user not supplying a BTF */
13853		if (!ret_btf) {
13854			verbose(env, "bpf_obj_new/bpf_percpu_obj_new requires prog BTF\n");
13855			return -EINVAL;
13856		}
13857
13858		ret_t = btf_type_by_id(ret_btf, ret_btf_id);
13859		if (!ret_t || !__btf_type_is_struct(ret_t)) {
13860			verbose(env, "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct\n");
13861			return -EINVAL;
13862		}
13863
13864		if (meta->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
13865			if (ret_t->size > BPF_GLOBAL_PERCPU_MA_MAX_SIZE) {
13866				verbose(env, "bpf_percpu_obj_new type size (%d) is greater than %d\n",
13867					ret_t->size, BPF_GLOBAL_PERCPU_MA_MAX_SIZE);
13868				return -EINVAL;
13869			}
13870
13871			if (!bpf_global_percpu_ma_set) {
13872				mutex_lock(&bpf_percpu_ma_lock);
13873				if (!bpf_global_percpu_ma_set) {
13874					/* Charge memory allocated with bpf_global_percpu_ma to
13875					 * root memcg. The obj_cgroup for root memcg is NULL.
13876					 */
13877					err = bpf_mem_alloc_percpu_init(&bpf_global_percpu_ma, NULL);
13878					if (!err)
13879						bpf_global_percpu_ma_set = true;
13880				}
13881				mutex_unlock(&bpf_percpu_ma_lock);
13882				if (err)
13883					return err;
13884			}
13885
13886			mutex_lock(&bpf_percpu_ma_lock);
13887			err = bpf_mem_alloc_percpu_unit_init(&bpf_global_percpu_ma, ret_t->size);
13888			mutex_unlock(&bpf_percpu_ma_lock);
13889			if (err)
13890				return err;
13891		}
13892
13893		struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id);
13894		if (meta->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
13895			if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) {
13896				verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n");
13897				return -EINVAL;
13898			}
13899
13900			if (struct_meta) {
13901				verbose(env, "bpf_percpu_obj_new type ID argument must not contain special fields\n");
13902				return -EINVAL;
13903			}
13904		}
13905
13906		mark_reg_known_zero(env, regs, BPF_REG_0);
13907		regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC;
13908		regs[BPF_REG_0].btf = ret_btf;
13909		regs[BPF_REG_0].btf_id = ret_btf_id;
13910		if (meta->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl])
13911			regs[BPF_REG_0].type |= MEM_PERCPU;
13912
13913		insn_aux->obj_new_size = ret_t->size;
13914		insn_aux->kptr_struct_meta = struct_meta;
13915	} else if (meta->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) {
13916		mark_reg_known_zero(env, regs, BPF_REG_0);
13917		regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC;
13918		regs[BPF_REG_0].btf = meta->arg_btf;
13919		regs[BPF_REG_0].btf_id = meta->arg_btf_id;
13920
13921		insn_aux->kptr_struct_meta =
13922			btf_find_struct_meta(meta->arg_btf,
13923					     meta->arg_btf_id);
13924	} else if (is_list_node_type(ptr_type)) {
13925		struct btf_field *field = meta->arg_list_head.field;
13926
13927		mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root);
13928	} else if (is_rbtree_node_type(ptr_type)) {
13929		struct btf_field *field = meta->arg_rbtree_root.field;
13930
13931		mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root);
13932	} else if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) {
13933		mark_reg_known_zero(env, regs, BPF_REG_0);
13934		regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED;
13935		regs[BPF_REG_0].btf = desc_btf;
13936		regs[BPF_REG_0].btf_id = meta->ret_btf_id;
13937	} else if (meta->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
13938		ret_t = btf_type_by_id(desc_btf, meta->arg_constant.value);
13939		if (!ret_t) {
13940			verbose(env, "Unknown type ID %lld passed to kfunc bpf_rdonly_cast\n",
13941				meta->arg_constant.value);
13942			return -EINVAL;
13943		} else if (btf_type_is_struct(ret_t)) {
13944			mark_reg_known_zero(env, regs, BPF_REG_0);
13945			regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_UNTRUSTED;
13946			regs[BPF_REG_0].btf = desc_btf;
13947			regs[BPF_REG_0].btf_id = meta->arg_constant.value;
13948		} else if (btf_type_is_void(ret_t)) {
13949			mark_reg_known_zero(env, regs, BPF_REG_0);
13950			regs[BPF_REG_0].type = PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED;
13951			regs[BPF_REG_0].mem_size = 0;
13952		} else {
13953			verbose(env,
13954				"kfunc bpf_rdonly_cast type ID argument must be of a struct or void\n");
13955			return -EINVAL;
13956		}
13957	} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice] ||
13958		   meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) {
13959		enum bpf_type_flag type_flag = get_dynptr_type_flag(meta->initialized_dynptr.type);
13960
13961		mark_reg_known_zero(env, regs, BPF_REG_0);
13962
13963		if (!meta->arg_constant.found) {
13964			verifier_bug(env, "bpf_dynptr_slice(_rdwr) no constant size");
13965			return -EFAULT;
13966		}
13967
13968		regs[BPF_REG_0].mem_size = meta->arg_constant.value;
13969
13970		/* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */
13971		regs[BPF_REG_0].type = PTR_TO_MEM | type_flag;
13972
13973		if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice]) {
13974			regs[BPF_REG_0].type |= MEM_RDONLY;
13975		} else {
13976			/* this will set env->seen_direct_write to true */
13977			if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) {
13978				verbose(env, "the prog does not allow writes to packet data\n");
13979				return -EINVAL;
13980			}
13981		}
13982
13983		if (!meta->initialized_dynptr.id) {
13984			verifier_bug(env, "no dynptr id");
13985			return -EFAULT;
13986		}
13987		regs[BPF_REG_0].dynptr_id = meta->initialized_dynptr.id;
13988
13989		/* we don't need to set BPF_REG_0's ref obj id
13990		 * because packet slices are not refcounted (see
13991		 * dynptr_type_refcounted)
13992		 */
13993	} else {
13994		return 0;
13995	}
13996
13997	return 1;
13998}
13999
14000static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name);
14001
14002static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
14003			    int *insn_idx_p)
14004{
14005	bool sleepable, rcu_lock, rcu_unlock, preempt_disable, preempt_enable;
14006	u32 i, nargs, ptr_type_id, release_ref_obj_id;
14007	struct bpf_reg_state *regs = cur_regs(env);
14008	const char *func_name, *ptr_type_name;
14009	const struct btf_type *t, *ptr_type;
14010	struct bpf_kfunc_call_arg_meta meta;
14011	struct bpf_insn_aux_data *insn_aux;
14012	int err, insn_idx = *insn_idx_p;
14013	const struct btf_param *args;
14014	struct btf *desc_btf;
14015
14016	/* skip for now, but return error when we find this in fixup_kfunc_call */
14017	if (!insn->imm)
14018		return 0;
14019
14020	err = fetch_kfunc_meta(env, insn, &meta, &func_name);
14021	if (err == -EACCES && func_name)
14022		verbose(env, "calling kernel function %s is not allowed\n", func_name);
14023	if (err)
14024		return err;
14025	desc_btf = meta.btf;
14026	insn_aux = &env->insn_aux_data[insn_idx];
14027
14028	insn_aux->is_iter_next = is_iter_next_kfunc(&meta);
14029
14030	if (!insn->off &&
14031	    (insn->imm == special_kfunc_list[KF_bpf_res_spin_lock] ||
14032	     insn->imm == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) {
14033		struct bpf_verifier_state *branch;
14034		struct bpf_reg_state *regs;
14035
14036		branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);
14037		if (IS_ERR(branch)) {
14038			verbose(env, "failed to push state for failed lock acquisition\n");
14039			return PTR_ERR(branch);
14040		}
14041
14042		regs = branch->frame[branch->curframe]->regs;
14043
14044		/* Clear r0-r5 registers in forked state */
14045		for (i = 0; i < CALLER_SAVED_REGS; i++)
14046			mark_reg_not_init(env, regs, caller_saved[i]);
14047
14048		mark_reg_unknown(env, regs, BPF_REG_0);
14049		err = __mark_reg_s32_range(env, regs, BPF_REG_0, -MAX_ERRNO, -1);
14050		if (err) {
14051			verbose(env, "failed to mark s32 range for retval in forked state for lock\n");
14052			return err;
14053		}
14054		__mark_btf_func_reg_size(env, regs, BPF_REG_0, sizeof(u32));
14055	} else if (!insn->off && insn->imm == special_kfunc_list[KF___bpf_trap]) {
14056		verbose(env, "unexpected __bpf_trap() due to uninitialized variable?\n");
14057		return -EFAULT;
14058	}
14059
14060	if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) {
14061		verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n");
14062		return -EACCES;
14063	}
14064
14065	sleepable = is_kfunc_sleepable(&meta);
14066	if (sleepable && !in_sleepable(env)) {
14067		verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name);
14068		return -EACCES;
14069	}
14070
14071	/* Track non-sleepable context for kfuncs, same as for helpers. */
14072	if (!in_sleepable_context(env))
14073		insn_aux->non_sleepable = true;
14074
14075	/* Check the arguments */
14076	err = check_kfunc_args(env, &meta, insn_idx);
14077	if (err < 0)
14078		return err;
14079
14080	if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
14081		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
14082					 set_rbtree_add_callback_state);
14083		if (err) {
14084			verbose(env, "kfunc %s#%d failed callback verification\n",
14085				func_name, meta.func_id);
14086			return err;
14087		}
14088	}
14089
14090	if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) {
14091		meta.r0_size = sizeof(u64);
14092		meta.r0_rdonly = false;
14093	}
14094
14095	if (is_bpf_wq_set_callback_impl_kfunc(meta.func_id)) {
14096		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
14097					 set_timer_callback_state);
14098		if (err) {
14099			verbose(env, "kfunc %s#%d failed callback verification\n",
14100				func_name, meta.func_id);
14101			return err;
14102		}
14103	}
14104
14105	if (is_task_work_add_kfunc(meta.func_id)) {
14106		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
14107					 set_task_work_schedule_callback_state);
14108		if (err) {
14109			verbose(env, "kfunc %s#%d failed callback verification\n",
14110				func_name, meta.func_id);
14111			return err;
14112		}
14113	}
14114
14115	rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta);
14116	rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta);
14117
14118	preempt_disable = is_kfunc_bpf_preempt_disable(&meta);
14119	preempt_enable = is_kfunc_bpf_preempt_enable(&meta);
14120
14121	if (rcu_lock) {
14122		env->cur_state->active_rcu_locks++;
14123	} else if (rcu_unlock) {
14124		struct bpf_func_state *state;
14125		struct bpf_reg_state *reg;
14126		u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER);
14127
14128		if (env->cur_state->active_rcu_locks == 0) {
14129			verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name);
14130			return -EINVAL;
14131		}
14132		if (--env->cur_state->active_rcu_locks == 0) {
14133			bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({
14134				if (reg->type & MEM_RCU) {
14135					reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
14136					reg->type |= PTR_UNTRUSTED;
14137				}
14138			}));
14139		}
14140	} else if (sleepable && env->cur_state->active_rcu_locks) {
14141		verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name);
14142		return -EACCES;
14143	}
14144
14145	if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) {
14146		verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n");
14147		return -EACCES;
14148	}
14149
14150	if (env->cur_state->active_preempt_locks) {
14151		if (preempt_disable) {
14152			env->cur_state->active_preempt_locks++;
14153		} else if (preempt_enable) {
14154			env->cur_state->active_preempt_locks--;
14155		} else if (sleepable) {
14156			verbose(env, "kernel func %s is sleepable within non-preemptible region\n", func_name);
14157			return -EACCES;
14158		}
14159	} else if (preempt_disable) {
14160		env->cur_state->active_preempt_locks++;
14161	} else if (preempt_enable) {
14162		verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name);
14163		return -EINVAL;
14164	}
14165
14166	if (env->cur_state->active_irq_id && sleepable) {
14167		verbose(env, "kernel func %s is sleepable within IRQ-disabled region\n", func_name);
14168		return -EACCES;
14169	}
14170
14171	if (is_kfunc_rcu_protected(&meta) && !in_rcu_cs(env)) {
14172		verbose(env, "kernel func %s requires RCU critical section protection\n", func_name);
14173		return -EACCES;
14174	}
14175
14176	/* In case of release function, we get register number of refcounted
14177	 * PTR_TO_BTF_ID in bpf_kfunc_arg_meta, do the release now.
14178	 */
14179	if (meta.release_regno) {
14180		struct bpf_reg_state *reg = &regs[meta.release_regno];
14181
14182		if (meta.initialized_dynptr.ref_obj_id) {
14183			err = unmark_stack_slots_dynptr(env, reg);
14184		} else {
14185			err = release_reference(env, reg->ref_obj_id);
14186			if (err)
14187				verbose(env, "kfunc %s#%d reference has not been acquired before\n",
14188					func_name, meta.func_id);
14189		}
14190		if (err)
14191			return err;
14192	}
14193
14194	if (meta.func_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
14195	    meta.func_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||
14196	    meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
14197		release_ref_obj_id = regs[BPF_REG_2].ref_obj_id;
14198		insn_aux->insert_off = regs[BPF_REG_2].off;
14199		insn_aux->kptr_struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id);
14200		err = ref_convert_owning_non_owning(env, release_ref_obj_id);
14201		if (err) {
14202			verbose(env, "kfunc %s#%d conversion of owning ref to non-owning failed\n",
14203				func_name, meta.func_id);
14204			return err;
14205		}
14206
14207		err = release_reference(env, release_ref_obj_id);
14208		if (err) {
14209			verbose(env, "kfunc %s#%d reference has not been acquired before\n",
14210				func_name, meta.func_id);
14211			return err;
14212		}
14213	}
14214
14215	if (meta.func_id == special_kfunc_list[KF_bpf_throw]) {
14216		if (!bpf_jit_supports_exceptions()) {
14217			verbose(env, "JIT does not support calling kfunc %s#%d\n",
14218				func_name, meta.func_id);
14219			return -ENOTSUPP;
14220		}
14221		env->seen_exception = true;
14222
14223		/* In the case of the default callback, the cookie value passed
14224		 * to bpf_throw becomes the return value of the program.
14225		 */
14226		if (!env->exception_callback_subprog) {
14227			err = check_return_code(env, BPF_REG_1, "R1");
14228			if (err < 0)
14229				return err;
14230		}
14231	}
14232
14233	for (i = 0; i < CALLER_SAVED_REGS; i++)
14234		mark_reg_not_init(env, regs, caller_saved[i]);
14235
14236	/* Check return type */
14237	t = btf_type_skip_modifiers(desc_btf, meta.func_proto->type, NULL);
14238
14239	if (is_kfunc_acquire(&meta) && !btf_type_is_struct_ptr(meta.btf, t)) {
14240		/* Only exception is bpf_obj_new_impl */
14241		if (meta.btf != btf_vmlinux ||
14242		    (meta.func_id != special_kfunc_list[KF_bpf_obj_new_impl] &&
14243		     meta.func_id != special_kfunc_list[KF_bpf_percpu_obj_new_impl] &&
14244		     meta.func_id != special_kfunc_list[KF_bpf_refcount_acquire_impl])) {
14245			verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n");
14246			return -EINVAL;
14247		}
14248	}
14249
14250	if (btf_type_is_scalar(t)) {
14251		mark_reg_unknown(env, regs, BPF_REG_0);
14252		if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] ||
14253		    meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]))
14254			__mark_reg_const_zero(env, &regs[BPF_REG_0]);
14255		mark_btf_func_reg_size(env, BPF_REG_0, t->size);
14256	} else if (btf_type_is_ptr(t)) {
14257		ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id);
14258		err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf);
14259		if (err) {
14260			if (err < 0)
14261				return err;
14262		} else if (btf_type_is_void(ptr_type)) {
14263			/* kfunc returning 'void *' is equivalent to returning scalar */
14264			mark_reg_unknown(env, regs, BPF_REG_0);
14265		} else if (!__btf_type_is_struct(ptr_type)) {
14266			if (!meta.r0_size) {
14267				__u32 sz;
14268
14269				if (!IS_ERR(btf_resolve_size(desc_btf, ptr_type, &sz))) {
14270					meta.r0_size = sz;
14271					meta.r0_rdonly = true;
14272				}
14273			}
14274			if (!meta.r0_size) {
14275				ptr_type_name = btf_name_by_offset(desc_btf,
14276								   ptr_type->name_off);
14277				verbose(env,
14278					"kernel function %s returns pointer type %s %s is not supported\n",
14279					func_name,
14280					btf_type_str(ptr_type),
14281					ptr_type_name);
14282				return -EINVAL;
14283			}
14284
14285			mark_reg_known_zero(env, regs, BPF_REG_0);
14286			regs[BPF_REG_0].type = PTR_TO_MEM;
14287			regs[BPF_REG_0].mem_size = meta.r0_size;
14288
14289			if (meta.r0_rdonly)
14290				regs[BPF_REG_0].type |= MEM_RDONLY;
14291
14292			/* Ensures we don't access the memory after a release_reference() */
14293			if (meta.ref_obj_id)
14294				regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
14295
14296			if (is_kfunc_rcu_protected(&meta))
14297				regs[BPF_REG_0].type |= MEM_RCU;
14298		} else {
14299			mark_reg_known_zero(env, regs, BPF_REG_0);
14300			regs[BPF_REG_0].btf = desc_btf;
14301			regs[BPF_REG_0].type = PTR_TO_BTF_ID;
14302			regs[BPF_REG_0].btf_id = ptr_type_id;
14303
14304			if (meta.func_id == special_kfunc_list[KF_bpf_get_kmem_cache])
14305				regs[BPF_REG_0].type |= PTR_UNTRUSTED;
14306			else if (is_kfunc_rcu_protected(&meta))
14307				regs[BPF_REG_0].type |= MEM_RCU;
14308
14309			if (is_iter_next_kfunc(&meta)) {
14310				struct bpf_reg_state *cur_iter;
14311
14312				cur_iter = get_iter_from_state(env->cur_state, &meta);
14313
14314				if (cur_iter->type & MEM_RCU) /* KF_RCU_PROTECTED */
14315					regs[BPF_REG_0].type |= MEM_RCU;
14316				else
14317					regs[BPF_REG_0].type |= PTR_TRUSTED;
14318			}
14319		}
14320
14321		if (is_kfunc_ret_null(&meta)) {
14322			regs[BPF_REG_0].type |= PTR_MAYBE_NULL;
14323			/* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */
14324			regs[BPF_REG_0].id = ++env->id_gen;
14325		}
14326		mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));
14327		if (is_kfunc_acquire(&meta)) {
14328			int id = acquire_reference(env, insn_idx);
14329
14330			if (id < 0)
14331				return id;
14332			if (is_kfunc_ret_null(&meta))
14333				regs[BPF_REG_0].id = id;
14334			regs[BPF_REG_0].ref_obj_id = id;
14335		} else if (is_rbtree_node_type(ptr_type) || is_list_node_type(ptr_type)) {
14336			ref_set_non_owning(env, &regs[BPF_REG_0]);
14337		}
14338
14339		if (reg_may_point_to_spin_lock(&regs[BPF_REG_0]) && !regs[BPF_REG_0].id)
14340			regs[BPF_REG_0].id = ++env->id_gen;
14341	} else if (btf_type_is_void(t)) {
14342		if (meta.btf == btf_vmlinux) {
14343			if (meta.func_id == special_kfunc_list[KF_bpf_obj_drop_impl] ||
14344			    meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl]) {
14345				insn_aux->kptr_struct_meta =
14346					btf_find_struct_meta(meta.arg_btf,
14347							     meta.arg_btf_id);
14348			}
14349		}
14350	}
14351
14352	if (is_kfunc_pkt_changing(&meta))
14353		clear_all_pkt_pointers(env);
14354
14355	nargs = btf_type_vlen(meta.func_proto);
14356	args = (const struct btf_param *)(meta.func_proto + 1);
14357	for (i = 0; i < nargs; i++) {
14358		u32 regno = i + 1;
14359
14360		t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL);
14361		if (btf_type_is_ptr(t))
14362			mark_btf_func_reg_size(env, regno, sizeof(void *));
14363		else
14364			/* scalar. ensured by btf_check_kfunc_arg_match() */
14365			mark_btf_func_reg_size(env, regno, t->size);
14366	}
14367
14368	if (is_iter_next_kfunc(&meta)) {
14369		err = process_iter_next_call(env, insn_idx, &meta);
14370		if (err)
14371			return err;
14372	}
14373
14374	return 0;
14375}
14376
14377static bool check_reg_sane_offset(struct bpf_verifier_env *env,
14378				  const struct bpf_reg_state *reg,
14379				  enum bpf_reg_type type)
14380{
14381	bool known = tnum_is_const(reg->var_off);
14382	s64 val = reg->var_off.value;
14383	s64 smin = reg->smin_value;
14384
14385	if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
14386		verbose(env, "math between %s pointer and %lld is not allowed\n",
14387			reg_type_str(env, type), val);
14388		return false;
14389	}
14390
14391	if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) {
14392		verbose(env, "%s pointer offset %d is not allowed\n",
14393			reg_type_str(env, type), reg->off);
14394		return false;
14395	}
14396
14397	if (smin == S64_MIN) {
14398		verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
14399			reg_type_str(env, type));
14400		return false;
14401	}
14402
14403	if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
14404		verbose(env, "value %lld makes %s pointer be out of bounds\n",
14405			smin, reg_type_str(env, type));
14406		return false;
14407	}
14408
14409	return true;
14410}
14411
14412enum {
14413	REASON_BOUNDS	= -1,
14414	REASON_TYPE	= -2,
14415	REASON_PATHS	= -3,
14416	REASON_LIMIT	= -4,
14417	REASON_STACK	= -5,
14418};
14419
14420static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
14421			      u32 *alu_limit, bool mask_to_left)
14422{
14423	u32 max = 0, ptr_limit = 0;
14424
14425	switch (ptr_reg->type) {
14426	case PTR_TO_STACK:
14427		/* Offset 0 is out-of-bounds, but acceptable start for the
14428		 * left direction, see BPF_REG_FP. Also, unknown scalar
14429		 * offset where we would need to deal with min/max bounds is
14430		 * currently prohibited for unprivileged.
14431		 */
14432		max = MAX_BPF_STACK + mask_to_left;
14433		ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off);
14434		break;
14435	case PTR_TO_MAP_VALUE:
14436		max = ptr_reg->map_ptr->value_size;
14437		ptr_limit = (mask_to_left ?
14438			     ptr_reg->smin_value :
14439			     ptr_reg->umax_value) + ptr_reg->off;
14440		break;
14441	default:
14442		return REASON_TYPE;
14443	}
14444
14445	if (ptr_limit >= max)
14446		return REASON_LIMIT;
14447	*alu_limit = ptr_limit;
14448	return 0;
14449}
14450
14451static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,
14452				    const struct bpf_insn *insn)
14453{
14454	return env->bypass_spec_v1 ||
14455		BPF_SRC(insn->code) == BPF_K ||
14456		cur_aux(env)->nospec;
14457}
14458
14459static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux,
14460				       u32 alu_state, u32 alu_limit)
14461{
14462	/* If we arrived here from different branches with different
14463	 * state or limits to sanitize, then this won't work.
14464	 */
14465	if (aux->alu_state &&
14466	    (aux->alu_state != alu_state ||
14467	     aux->alu_limit != alu_limit))
14468		return REASON_PATHS;
14469
14470	/* Corresponding fixup done in do_misc_fixups(). */
14471	aux->alu_state = alu_state;
14472	aux->alu_limit = alu_limit;
14473	return 0;
14474}
14475
14476static int sanitize_val_alu(struct bpf_verifier_env *env,
14477			    struct bpf_insn *insn)
14478{
14479	struct bpf_insn_aux_data *aux = cur_aux(env);
14480
14481	if (can_skip_alu_sanitation(env, insn))
14482		return 0;
14483
14484	return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0);
14485}
14486
14487static bool sanitize_needed(u8 opcode)
14488{
14489	return opcode == BPF_ADD || opcode == BPF_SUB;
14490}
14491
14492struct bpf_sanitize_info {
14493	struct bpf_insn_aux_data aux;
14494	bool mask_to_left;
14495};
14496
14497static int sanitize_speculative_path(struct bpf_verifier_env *env,
14498				     const struct bpf_insn *insn,
14499				     u32 next_idx, u32 curr_idx)
14500{
14501	struct bpf_verifier_state *branch;
14502	struct bpf_reg_state *regs;
14503
14504	branch = push_stack(env, next_idx, curr_idx, true);
14505	if (!IS_ERR(branch) && insn) {
14506		regs = branch->frame[branch->curframe]->regs;
14507		if (BPF_SRC(insn->code) == BPF_K) {
14508			mark_reg_unknown(env, regs, insn->dst_reg);
14509		} else if (BPF_SRC(insn->code) == BPF_X) {
14510			mark_reg_unknown(env, regs, insn->dst_reg);
14511			mark_reg_unknown(env, regs, insn->src_reg);
14512		}
14513	}
14514	return PTR_ERR_OR_ZERO(branch);
14515}
14516
14517static int sanitize_ptr_alu(struct bpf_verifier_env *env,
14518			    struct bpf_insn *insn,
14519			    const struct bpf_reg_state *ptr_reg,
14520			    const struct bpf_reg_state *off_reg,
14521			    struct bpf_reg_state *dst_reg,
14522			    struct bpf_sanitize_info *info,
14523			    const bool commit_window)
14524{
14525	struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux;
14526	struct bpf_verifier_state *vstate = env->cur_state;
14527	bool off_is_imm = tnum_is_const(off_reg->var_off);
14528	bool off_is_neg = off_reg->smin_value < 0;
14529	bool ptr_is_dst_reg = ptr_reg == dst_reg;
14530	u8 opcode = BPF_OP(insn->code);
14531	u32 alu_state, alu_limit;
14532	struct bpf_reg_state tmp;
14533	int err;
14534
14535	if (can_skip_alu_sanitation(env, insn))
14536		return 0;
14537
14538	/* We already marked aux for masking from non-speculative
14539	 * paths, thus we got here in the first place. We only care
14540	 * to explore bad access from here.
14541	 */
14542	if (vstate->speculative)
14543		goto do_sim;
14544
14545	if (!commit_window) {
14546		if (!tnum_is_const(off_reg->var_off) &&
14547		    (off_reg->smin_value < 0) != (off_reg->smax_value < 0))
14548			return REASON_BOUNDS;
14549
14550		info->mask_to_left = (opcode == BPF_ADD &&  off_is_neg) ||
14551				     (opcode == BPF_SUB && !off_is_neg);
14552	}
14553
14554	err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left);
14555	if (err < 0)
14556		return err;
14557
14558	if (commit_window) {
14559		/* In commit phase we narrow the masking window based on
14560		 * the observed pointer move after the simulated operation.
14561		 */
14562		alu_state = info->aux.alu_state;
14563		alu_limit = abs(info->aux.alu_limit - alu_limit);
14564	} else {
14565		alu_state  = off_is_neg ? BPF_ALU_NEG_VALUE : 0;
14566		alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0;
14567		alu_state |= ptr_is_dst_reg ?
14568			     BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;
14569
14570		/* Limit pruning on unknown scalars to enable deep search for
14571		 * potential masking differences from other program paths.
14572		 */
14573		if (!off_is_imm)
14574			env->explore_alu_limits = true;
14575	}
14576
14577	err = update_alu_sanitation_state(aux, alu_state, alu_limit);
14578	if (err < 0)
14579		return err;
14580do_sim:
14581	/* If we're in commit phase, we're done here given we already
14582	 * pushed the truncated dst_reg into the speculative verification
14583	 * stack.
14584	 *
14585	 * Also, when register is a known constant, we rewrite register-based
14586	 * operation to immediate-based, and thus do not need masking (and as
14587	 * a consequence, do not need to simulate the zero-truncation either).
14588	 */
14589	if (commit_window || off_is_imm)
14590		return 0;
14591
14592	/* Simulate and find potential out-of-bounds access under
14593	 * speculative execution from truncation as a result of
14594	 * masking when off was not within expected range. If off
14595	 * sits in dst, then we temporarily need to move ptr there
14596	 * to simulate dst (== 0) +/-= ptr. Needed, for example,
14597	 * for cases where we use K-based arithmetic in one direction
14598	 * and truncated reg-based in the other in order to explore
14599	 * bad access.
14600	 */
14601	if (!ptr_is_dst_reg) {
14602		tmp = *dst_reg;
14603		copy_register_state(dst_reg, ptr_reg);
14604	}
14605	err = sanitize_speculative_path(env, NULL, env->insn_idx + 1, env->insn_idx);
14606	if (err < 0)
14607		return REASON_STACK;
14608	if (!ptr_is_dst_reg)
14609		*dst_reg = tmp;
14610	return 0;
14611}
14612
14613static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)
14614{
14615	struct bpf_verifier_state *vstate = env->cur_state;
14616
14617	/* If we simulate paths under speculation, we don't update the
14618	 * insn as 'seen' such that when we verify unreachable paths in
14619	 * the non-speculative domain, sanitize_dead_code() can still
14620	 * rewrite/sanitize them.
14621	 */
14622	if (!vstate->speculative)
14623		env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;
14624}
14625
14626static int sanitize_err(struct bpf_verifier_env *env,
14627			const struct bpf_insn *insn, int reason,
14628			const struct bpf_reg_state *off_reg,
14629			const struct bpf_reg_state *dst_reg)
14630{
14631	static const char *err = "pointer arithmetic with it prohibited for !root";
14632	const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub";
14633	u32 dst = insn->dst_reg, src = insn->src_reg;
14634
14635	switch (reason) {
14636	case REASON_BOUNDS:
14637		verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n",
14638			off_reg == dst_reg ? dst : src, err);
14639		break;
14640	case REASON_TYPE:
14641		verbose(env, "R%d has pointer with unsupported alu operation, %s\n",
14642			off_reg == dst_reg ? src : dst, err);
14643		break;
14644	case REASON_PATHS:
14645		verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n",
14646			dst, op, err);
14647		break;
14648	case REASON_LIMIT:
14649		verbose(env, "R%d tried to %s beyond pointer bounds, %s\n",
14650			dst, op, err);
14651		break;
14652	case REASON_STACK:
14653		verbose(env, "R%d could not be pushed for speculative verification, %s\n",
14654			dst, err);
14655		return -ENOMEM;
14656	default:
14657		verifier_bug(env, "unknown reason (%d)", reason);
14658		break;
14659	}
14660
14661	return -EACCES;
14662}
14663
14664/* check that stack access falls within stack limits and that 'reg' doesn't
14665 * have a variable offset.
14666 *
14667 * Variable offset is prohibited for unprivileged mode for simplicity since it
14668 * requires corresponding support in Spectre masking for stack ALU.  See also
14669 * retrieve_ptr_limit().
14670 *
14671 *
14672 * 'off' includes 'reg->off'.
14673 */
14674static int check_stack_access_for_ptr_arithmetic(
14675				struct bpf_verifier_env *env,
14676				int regno,
14677				const struct bpf_reg_state *reg,
14678				int off)
14679{
14680	if (!tnum_is_const(reg->var_off)) {
14681		char tn_buf[48];
14682
14683		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
14684		verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n",
14685			regno, tn_buf, off);
14686		return -EACCES;
14687	}
14688
14689	if (off >= 0 || off < -MAX_BPF_STACK) {
14690		verbose(env, "R%d stack pointer arithmetic goes out of range, "
14691			"prohibited for !root; off=%d\n", regno, off);
14692		return -EACCES;
14693	}
14694
14695	return 0;
14696}
14697
14698static int sanitize_check_bounds(struct bpf_verifier_env *env,
14699				 const struct bpf_insn *insn,
14700				 const struct bpf_reg_state *dst_reg)
14701{
14702	u32 dst = insn->dst_reg;
14703
14704	/* For unprivileged we require that resulting offset must be in bounds
14705	 * in order to be able to sanitize access later on.
14706	 */
14707	if (env->bypass_spec_v1)
14708		return 0;
14709
14710	switch (dst_reg->type) {
14711	case PTR_TO_STACK:
14712		if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg,
14713					dst_reg->off + dst_reg->var_off.value))
14714			return -EACCES;
14715		break;
14716	case PTR_TO_MAP_VALUE:
14717		if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) {
14718			verbose(env, "R%d pointer arithmetic of map value goes out of range, "
14719				"prohibited for !root\n", dst);
14720			return -EACCES;
14721		}
14722		break;
14723	default:
14724		return -EOPNOTSUPP;
14725	}
14726
14727	return 0;
14728}
14729
14730/* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off.
14731 * Caller should also handle BPF_MOV case separately.
14732 * If we return -EACCES, caller may want to try again treating pointer as a
14733 * scalar.  So we only emit a diagnostic if !env->allow_ptr_leaks.
14734 */
14735static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
14736				   struct bpf_insn *insn,
14737				   const struct bpf_reg_state *ptr_reg,
14738				   const struct bpf_reg_state *off_reg)
14739{
14740	struct bpf_verifier_state *vstate = env->cur_state;
14741	struct bpf_func_state *state = vstate->frame[vstate->curframe];
14742	struct bpf_reg_state *regs = state->regs, *dst_reg;
14743	bool known = tnum_is_const(off_reg->var_off);
14744	s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value,
14745	    smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value;
14746	u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value,
14747	    umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value;
14748	struct bpf_sanitize_info info = {};
14749	u8 opcode = BPF_OP(insn->code);
14750	u32 dst = insn->dst_reg;
14751	int ret, bounds_ret;
14752
14753	dst_reg = &regs[dst];
14754
14755	if ((known && (smin_val != smax_val || umin_val != umax_val)) ||
14756	    smin_val > smax_val || umin_val > umax_val) {
14757		/* Taint dst register if offset had invalid bounds derived from
14758		 * e.g. dead branches.
14759		 */
14760		__mark_reg_unknown(env, dst_reg);
14761		return 0;
14762	}
14763
14764	if (BPF_CLASS(insn->code) != BPF_ALU64) {
14765		/* 32-bit ALU ops on pointers produce (meaningless) scalars */
14766		if (opcode == BPF_SUB && env->allow_ptr_leaks) {
14767			__mark_reg_unknown(env, dst_reg);
14768			return 0;
14769		}
14770
14771		verbose(env,
14772			"R%d 32-bit pointer arithmetic prohibited\n",
14773			dst);
14774		return -EACCES;
14775	}
14776
14777	if (ptr_reg->type & PTR_MAYBE_NULL) {
14778		verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
14779			dst, reg_type_str(env, ptr_reg->type));
14780		return -EACCES;
14781	}
14782
14783	/*
14784	 * Accesses to untrusted PTR_TO_MEM are done through probe
14785	 * instructions, hence no need to track offsets.
14786	 */
14787	if (base_type(ptr_reg->type) == PTR_TO_MEM && (ptr_reg->type & PTR_UNTRUSTED))
14788		return 0;
14789
14790	switch (base_type(ptr_reg->type)) {
14791	case PTR_TO_CTX:
14792	case PTR_TO_MAP_VALUE:
14793	case PTR_TO_MAP_KEY:
14794	case PTR_TO_STACK:
14795	case PTR_TO_PACKET_META:
14796	case PTR_TO_PACKET:
14797	case PTR_TO_TP_BUFFER:
14798	case PTR_TO_BTF_ID:
14799	case PTR_TO_MEM:
14800	case PTR_TO_BUF:
14801	case PTR_TO_FUNC:
14802	case CONST_PTR_TO_DYNPTR:
14803		break;
14804	case PTR_TO_FLOW_KEYS:
14805		if (known)
14806			break;
14807		fallthrough;
14808	case CONST_PTR_TO_MAP:
14809		/* smin_val represents the known value */
14810		if (known && smin_val == 0 && opcode == BPF_ADD)
14811			break;
14812		fallthrough;
14813	default:
14814		verbose(env, "R%d pointer arithmetic on %s prohibited\n",
14815			dst, reg_type_str(env, ptr_reg->type));
14816		return -EACCES;
14817	}
14818
14819	/* In case of 'scalar += pointer', dst_reg inherits pointer type and id.
14820	 * The id may be overwritten later if we create a new variable offset.
14821	 */
14822	dst_reg->type = ptr_reg->type;
14823	dst_reg->id = ptr_reg->id;
14824
14825	if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) ||
14826	    !check_reg_sane_offset(env, ptr_reg, ptr_reg->type))
14827		return -EINVAL;
14828
14829	/* pointer types do not carry 32-bit bounds at the moment. */
14830	__mark_reg32_unbounded(dst_reg);
14831
14832	if (sanitize_needed(opcode)) {
14833		ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,
14834				       &info, false);
14835		if (ret < 0)
14836			return sanitize_err(env, insn, ret, off_reg, dst_reg);
14837	}
14838
14839	switch (opcode) {
14840	case BPF_ADD:
14841		/* We can take a fixed offset as long as it doesn't overflow
14842		 * the s32 'off' field
14843		 */
14844		if (known && (ptr_reg->off + smin_val ==
14845			      (s64)(s32)(ptr_reg->off + smin_val))) {
14846			/* pointer += K.  Accumulate it into fixed offset */
14847			dst_reg->smin_value = smin_ptr;
14848			dst_reg->smax_value = smax_ptr;
14849			dst_reg->umin_value = umin_ptr;
14850			dst_reg->umax_value = umax_ptr;
14851			dst_reg->var_off = ptr_reg->var_off;
14852			dst_reg->off = ptr_reg->off + smin_val;
14853			dst_reg->raw = ptr_reg->raw;
14854			break;
14855		}
14856		/* A new variable offset is created.  Note that off_reg->off
14857		 * == 0, since it's a scalar.
14858		 * dst_reg gets the pointer type and since some positive
14859		 * integer value was added to the pointer, give it a new 'id'
14860		 * if it's a PTR_TO_PACKET.
14861		 * this creates a new 'base' pointer, off_reg (variable) gets
14862		 * added into the variable offset, and we copy the fixed offset
14863		 * from ptr_reg.
14864		 */
14865		if (check_add_overflow(smin_ptr, smin_val, &dst_reg->smin_value) ||
14866		    check_add_overflow(smax_ptr, smax_val, &dst_reg->smax_value)) {
14867			dst_reg->smin_value = S64_MIN;
14868			dst_reg->smax_value = S64_MAX;
14869		}
14870		if (check_add_overflow(umin_ptr, umin_val, &dst_reg->umin_value) ||
14871		    check_add_overflow(umax_ptr, umax_val, &dst_reg->umax_value)) {
14872			dst_reg->umin_value = 0;
14873			dst_reg->umax_value = U64_MAX;
14874		}
14875		dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off);
14876		dst_reg->off = ptr_reg->off;
14877		dst_reg->raw = ptr_reg->raw;
14878		if (reg_is_pkt_pointer(ptr_reg)) {
14879			dst_reg->id = ++env->id_gen;
14880			/* something was added to pkt_ptr, set range to zero */
14881			memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
14882		}
14883		break;
14884	case BPF_SUB:
14885		if (dst_reg == off_reg) {
14886			/* scalar -= pointer.  Creates an unknown scalar */
14887			verbose(env, "R%d tried to subtract pointer from scalar\n",
14888				dst);
14889			return -EACCES;
14890		}
14891		/* We don't allow subtraction from FP, because (according to
14892		 * test_verifier.c test "invalid fp arithmetic", JITs might not
14893		 * be able to deal with it.
14894		 */
14895		if (ptr_reg->type == PTR_TO_STACK) {
14896			verbose(env, "R%d subtraction from stack pointer prohibited\n",
14897				dst);
14898			return -EACCES;
14899		}
14900		if (known && (ptr_reg->off - smin_val ==
14901			      (s64)(s32)(ptr_reg->off - smin_val))) {
14902			/* pointer -= K.  Subtract it from fixed offset */
14903			dst_reg->smin_value = smin_ptr;
14904			dst_reg->smax_value = smax_ptr;
14905			dst_reg->umin_value = umin_ptr;
14906			dst_reg->umax_value = umax_ptr;
14907			dst_reg->var_off = ptr_reg->var_off;
14908			dst_reg->id = ptr_reg->id;
14909			dst_reg->off = ptr_reg->off - smin_val;
14910			dst_reg->raw = ptr_reg->raw;
14911			break;
14912		}
14913		/* A new variable offset is created.  If the subtrahend is known
14914		 * nonnegative, then any reg->range we had before is still good.
14915		 */
14916		if (check_sub_overflow(smin_ptr, smax_val, &dst_reg->smin_value) ||
14917		    check_sub_overflow(smax_ptr, smin_val, &dst_reg->smax_value)) {
14918			/* Overflow possible, we know nothing */
14919			dst_reg->smin_value = S64_MIN;
14920			dst_reg->smax_value = S64_MAX;
14921		}
14922		if (umin_ptr < umax_val) {
14923			/* Overflow possible, we know nothing */
14924			dst_reg->umin_value = 0;
14925			dst_reg->umax_value = U64_MAX;
14926		} else {
14927			/* Cannot overflow (as long as bounds are consistent) */
14928			dst_reg->umin_value = umin_ptr - umax_val;
14929			dst_reg->umax_value = umax_ptr - umin_val;
14930		}
14931		dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off);
14932		dst_reg->off = ptr_reg->off;
14933		dst_reg->raw = ptr_reg->raw;
14934		if (reg_is_pkt_pointer(ptr_reg)) {
14935			dst_reg->id = ++env->id_gen;
14936			/* something was added to pkt_ptr, set range to zero */
14937			if (smin_val < 0)
14938				memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
14939		}
14940		break;
14941	case BPF_AND:
14942	case BPF_OR:
14943	case BPF_XOR:
14944		/* bitwise ops on pointers are troublesome, prohibit. */
14945		verbose(env, "R%d bitwise operator %s on pointer prohibited\n",
14946			dst, bpf_alu_string[opcode >> 4]);
14947		return -EACCES;
14948	default:
14949		/* other operators (e.g. MUL,LSH) produce non-pointer results */
14950		verbose(env, "R%d pointer arithmetic with %s operator prohibited\n",
14951			dst, bpf_alu_string[opcode >> 4]);
14952		return -EACCES;
14953	}
14954
14955	if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type))
14956		return -EINVAL;
14957	reg_bounds_sync(dst_reg);
14958	bounds_ret = sanitize_check_bounds(env, insn, dst_reg);
14959	if (bounds_ret == -EACCES)
14960		return bounds_ret;
14961	if (sanitize_needed(opcode)) {
14962		ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg,
14963				       &info, true);
14964		if (verifier_bug_if(!can_skip_alu_sanitation(env, insn)
14965				    && !env->cur_state->speculative
14966				    && bounds_ret
14967				    && !ret,
14968				    env, "Pointer type unsupported by sanitize_check_bounds() not rejected by retrieve_ptr_limit() as required")) {
14969			return -EFAULT;
14970		}
14971		if (ret < 0)
14972			return sanitize_err(env, insn, ret, off_reg, dst_reg);
14973	}
14974
14975	return 0;
14976}
14977
14978static void scalar32_min_max_add(struct bpf_reg_state *dst_reg,
14979				 struct bpf_reg_state *src_reg)
14980{
14981	s32 *dst_smin = &dst_reg->s32_min_value;
14982	s32 *dst_smax = &dst_reg->s32_max_value;
14983	u32 *dst_umin = &dst_reg->u32_min_value;
14984	u32 *dst_umax = &dst_reg->u32_max_value;
14985	u32 umin_val = src_reg->u32_min_value;
14986	u32 umax_val = src_reg->u32_max_value;
14987	bool min_overflow, max_overflow;
14988
14989	if (check_add_overflow(*dst_smin, src_reg->s32_min_value, dst_smin) ||
14990	    check_add_overflow(*dst_smax, src_reg->s32_max_value, dst_smax)) {
14991		*dst_smin = S32_MIN;
14992		*dst_smax = S32_MAX;
14993	}
14994
14995	/* If either all additions overflow or no additions overflow, then
14996	 * it is okay to set: dst_umin = dst_umin + src_umin, dst_umax =
14997	 * dst_umax + src_umax. Otherwise (some additions overflow), set
14998	 * the output bounds to unbounded.
14999	 */
15000	min_overflow = check_add_overflow(*dst_umin, umin_val, dst_umin);
15001	max_overflow = check_add_overflow(*dst_umax, umax_val, dst_umax);
15002
15003	if (!min_overflow && max_overflow) {
15004		*dst_umin = 0;
15005		*dst_umax = U32_MAX;
15006	}
15007}
15008
15009static void scalar_min_max_add(struct bpf_reg_state *dst_reg,
15010			       struct bpf_reg_state *src_reg)
15011{
15012	s64 *dst_smin = &dst_reg->smin_value;
15013	s64 *dst_smax = &dst_reg->smax_value;
15014	u64 *dst_umin = &dst_reg->umin_value;
15015	u64 *dst_umax = &dst_reg->umax_value;
15016	u64 umin_val = src_reg->umin_value;
15017	u64 umax_val = src_reg->umax_value;
15018	bool min_overflow, max_overflow;
15019
15020	if (check_add_overflow(*dst_smin, src_reg->smin_value, dst_smin) ||
15021	    check_add_overflow(*dst_smax, src_reg->smax_value, dst_smax)) {
15022		*dst_smin = S64_MIN;
15023		*dst_smax = S64_MAX;
15024	}
15025
15026	/* If either all additions overflow or no additions overflow, then
15027	 * it is okay to set: dst_umin = dst_umin + src_umin, dst_umax =
15028	 * dst_umax + src_umax. Otherwise (some additions overflow), set
15029	 * the output bounds to unbounded.
15030	 */
15031	min_overflow = check_add_overflow(*dst_umin, umin_val, dst_umin);
15032	max_overflow = check_add_overflow(*dst_umax, umax_val, dst_umax);
15033
15034	if (!min_overflow && max_overflow) {
15035		*dst_umin = 0;
15036		*dst_umax = U64_MAX;
15037	}
15038}
15039
15040static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg,
15041				 struct bpf_reg_state *src_reg)
15042{
15043	s32 *dst_smin = &dst_reg->s32_min_value;
15044	s32 *dst_smax = &dst_reg->s32_max_value;
15045	u32 *dst_umin = &dst_reg->u32_min_value;
15046	u32 *dst_umax = &dst_reg->u32_max_value;
15047	u32 umin_val = src_reg->u32_min_value;
15048	u32 umax_val = src_reg->u32_max_value;
15049	bool min_underflow, max_underflow;
15050
15051	if (check_sub_overflow(*dst_smin, src_reg->s32_max_value, dst_smin) ||
15052	    check_sub_overflow(*dst_smax, src_reg->s32_min_value, dst_smax)) {
15053		/* Overflow possible, we know nothing */
15054		*dst_smin = S32_MIN;
15055		*dst_smax = S32_MAX;
15056	}
15057
15058	/* If either all subtractions underflow or no subtractions
15059	 * underflow, it is okay to set: dst_umin = dst_umin - src_umax,
15060	 * dst_umax = dst_umax - src_umin. Otherwise (some subtractions
15061	 * underflow), set the output bounds to unbounded.
15062	 */
15063	min_underflow = check_sub_overflow(*dst_umin, umax_val, dst_umin);
15064	max_underflow = check_sub_overflow(*dst_umax, umin_val, dst_umax);
15065
15066	if (min_underflow && !max_underflow) {
15067		*dst_umin = 0;
15068		*dst_umax = U32_MAX;
15069	}
15070}
15071
15072static void scalar_min_max_sub(struct bpf_reg_state *dst_reg,
15073			       struct bpf_reg_state *src_reg)
15074{
15075	s64 *dst_smin = &dst_reg->smin_value;
15076	s64 *dst_smax = &dst_reg->smax_value;
15077	u64 *dst_umin = &dst_reg->umin_value;
15078	u64 *dst_umax = &dst_reg->umax_value;
15079	u64 umin_val = src_reg->umin_value;
15080	u64 umax_val = src_reg->umax_value;
15081	bool min_underflow, max_underflow;
15082
15083	if (check_sub_overflow(*dst_smin, src_reg->smax_value, dst_smin) ||
15084	    check_sub_overflow(*dst_smax, src_reg->smin_value, dst_smax)) {
15085		/* Overflow possible, we know nothing */
15086		*dst_smin = S64_MIN;
15087		*dst_smax = S64_MAX;
15088	}
15089
15090	/* If either all subtractions underflow or no subtractions
15091	 * underflow, it is okay to set: dst_umin = dst_umin - src_umax,
15092	 * dst_umax = dst_umax - src_umin. Otherwise (some subtractions
15093	 * underflow), set the output bounds to unbounded.
15094	 */
15095	min_underflow = check_sub_overflow(*dst_umin, umax_val, dst_umin);
15096	max_underflow = check_sub_overflow(*dst_umax, umin_val, dst_umax);
15097
15098	if (min_underflow && !max_underflow) {
15099		*dst_umin = 0;
15100		*dst_umax = U64_MAX;
15101	}
15102}
15103
15104static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg,
15105				 struct bpf_reg_state *src_reg)
15106{
15107	s32 *dst_smin = &dst_reg->s32_min_value;
15108	s32 *dst_smax = &dst_reg->s32_max_value;
15109	u32 *dst_umin = &dst_reg->u32_min_value;
15110	u32 *dst_umax = &dst_reg->u32_max_value;
15111	s32 tmp_prod[4];
15112
15113	if (check_mul_overflow(*dst_umax, src_reg->u32_max_value, dst_umax) ||
15114	    check_mul_overflow(*dst_umin, src_reg->u32_min_value, dst_umin)) {
15115		/* Overflow possible, we know nothing */
15116		*dst_umin = 0;
15117		*dst_umax = U32_MAX;
15118	}
15119	if (check_mul_overflow(*dst_smin, src_reg->s32_min_value, &tmp_prod[0]) ||
15120	    check_mul_overflow(*dst_smin, src_reg->s32_max_value, &tmp_prod[1]) ||
15121	    check_mul_overflow(*dst_smax, src_reg->s32_min_value, &tmp_prod[2]) ||
15122	    check_mul_overflow(*dst_smax, src_reg->s32_max_value, &tmp_prod[3])) {
15123		/* Overflow possible, we know nothing */
15124		*dst_smin = S32_MIN;
15125		*dst_smax = S32_MAX;
15126	} else {
15127		*dst_smin = min_array(tmp_prod, 4);
15128		*dst_smax = max_array(tmp_prod, 4);
15129	}
15130}
15131
15132static void scalar_min_max_mul(struct bpf_reg_state *dst_reg,
15133			       struct bpf_reg_state *src_reg)
15134{
15135	s64 *dst_smin = &dst_reg->smin_value;
15136	s64 *dst_smax = &dst_reg->smax_value;
15137	u64 *dst_umin = &dst_reg->umin_value;
15138	u64 *dst_umax = &dst_reg->umax_value;
15139	s64 tmp_prod[4];
15140
15141	if (check_mul_overflow(*dst_umax, src_reg->umax_value, dst_umax) ||
15142	    check_mul_overflow(*dst_umin, src_reg->umin_value, dst_umin)) {
15143		/* Overflow possible, we know nothing */
15144		*dst_umin = 0;
15145		*dst_umax = U64_MAX;
15146	}
15147	if (check_mul_overflow(*dst_smin, src_reg->smin_value, &tmp_prod[0]) ||
15148	    check_mul_overflow(*dst_smin, src_reg->smax_value, &tmp_prod[1]) ||
15149	    check_mul_overflow(*dst_smax, src_reg->smin_value, &tmp_prod[2]) ||
15150	    check_mul_overflow(*dst_smax, src_reg->smax_value, &tmp_prod[3])) {
15151		/* Overflow possible, we know nothing */
15152		*dst_smin = S64_MIN;
15153		*dst_smax = S64_MAX;
15154	} else {
15155		*dst_smin = min_array(tmp_prod, 4);
15156		*dst_smax = max_array(tmp_prod, 4);
15157	}
15158}
15159
15160static void scalar32_min_max_and(struct bpf_reg_state *dst_reg,
15161				 struct bpf_reg_state *src_reg)
15162{
15163	bool src_known = tnum_subreg_is_const(src_reg->var_off);
15164	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
15165	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
15166	u32 umax_val = src_reg->u32_max_value;
15167
15168	if (src_known && dst_known) {
15169		__mark_reg32_known(dst_reg, var32_off.value);
15170		return;
15171	}
15172
15173	/* We get our minimum from the var_off, since that's inherently
15174	 * bitwise.  Our maximum is the minimum of the operands' maxima.
15175	 */
15176	dst_reg->u32_min_value = var32_off.value;
15177	dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val);
15178
15179	/* Safe to set s32 bounds by casting u32 result into s32 when u32
15180	 * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded.
15181	 */
15182	if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) {
15183		dst_reg->s32_min_value = dst_reg->u32_min_value;
15184		dst_reg->s32_max_value = dst_reg->u32_max_value;
15185	} else {
15186		dst_reg->s32_min_value = S32_MIN;
15187		dst_reg->s32_max_value = S32_MAX;
15188	}
15189}
15190
15191static void scalar_min_max_and(struct bpf_reg_state *dst_reg,
15192			       struct bpf_reg_state *src_reg)
15193{
15194	bool src_known = tnum_is_const(src_reg->var_off);
15195	bool dst_known = tnum_is_const(dst_reg->var_off);
15196	u64 umax_val = src_reg->umax_value;
15197
15198	if (src_known && dst_known) {
15199		__mark_reg_known(dst_reg, dst_reg->var_off.value);
15200		return;
15201	}
15202
15203	/* We get our minimum from the var_off, since that's inherently
15204	 * bitwise.  Our maximum is the minimum of the operands' maxima.
15205	 */
15206	dst_reg->umin_value = dst_reg->var_off.value;
15207	dst_reg->umax_value = min(dst_reg->umax_value, umax_val);
15208
15209	/* Safe to set s64 bounds by casting u64 result into s64 when u64
15210	 * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded.
15211	 */
15212	if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) {
15213		dst_reg->smin_value = dst_reg->umin_value;
15214		dst_reg->smax_value = dst_reg->umax_value;
15215	} else {
15216		dst_reg->smin_value = S64_MIN;
15217		dst_reg->smax_value = S64_MAX;
15218	}
15219	/* We may learn something more from the var_off */
15220	__update_reg_bounds(dst_reg);
15221}
15222
15223static void scalar32_min_max_or(struct bpf_reg_state *dst_reg,
15224				struct bpf_reg_state *src_reg)
15225{
15226	bool src_known = tnum_subreg_is_const(src_reg->var_off);
15227	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
15228	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
15229	u32 umin_val = src_reg->u32_min_value;
15230
15231	if (src_known && dst_known) {
15232		__mark_reg32_known(dst_reg, var32_off.value);
15233		return;
15234	}
15235
15236	/* We get our maximum from the var_off, and our minimum is the
15237	 * maximum of the operands' minima
15238	 */
15239	dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val);
15240	dst_reg->u32_max_value = var32_off.value | var32_off.mask;
15241
15242	/* Safe to set s32 bounds by casting u32 result into s32 when u32
15243	 * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded.
15244	 */
15245	if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) {
15246		dst_reg->s32_min_value = dst_reg->u32_min_value;
15247		dst_reg->s32_max_value = dst_reg->u32_max_value;
15248	} else {
15249		dst_reg->s32_min_value = S32_MIN;
15250		dst_reg->s32_max_value = S32_MAX;
15251	}
15252}
15253
15254static void scalar_min_max_or(struct bpf_reg_state *dst_reg,
15255			      struct bpf_reg_state *src_reg)
15256{
15257	bool src_known = tnum_is_const(src_reg->var_off);
15258	bool dst_known = tnum_is_const(dst_reg->var_off);
15259	u64 umin_val = src_reg->umin_value;
15260
15261	if (src_known && dst_known) {
15262		__mark_reg_known(dst_reg, dst_reg->var_off.value);
15263		return;
15264	}
15265
15266	/* We get our maximum from the var_off, and our minimum is the
15267	 * maximum of the operands' minima
15268	 */
15269	dst_reg->umin_value = max(dst_reg->umin_value, umin_val);
15270	dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask;
15271
15272	/* Safe to set s64 bounds by casting u64 result into s64 when u64
15273	 * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded.
15274	 */
15275	if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) {
15276		dst_reg->smin_value = dst_reg->umin_value;
15277		dst_reg->smax_value = dst_reg->umax_value;
15278	} else {
15279		dst_reg->smin_value = S64_MIN;
15280		dst_reg->smax_value = S64_MAX;
15281	}
15282	/* We may learn something more from the var_off */
15283	__update_reg_bounds(dst_reg);
15284}
15285
15286static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg,
15287				 struct bpf_reg_state *src_reg)
15288{
15289	bool src_known = tnum_subreg_is_const(src_reg->var_off);
15290	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
15291	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
15292
15293	if (src_known && dst_known) {
15294		__mark_reg32_known(dst_reg, var32_off.value);
15295		return;
15296	}
15297
15298	/* We get both minimum and maximum from the var32_off. */
15299	dst_reg->u32_min_value = var32_off.value;
15300	dst_reg->u32_max_value = var32_off.value | var32_off.mask;
15301
15302	/* Safe to set s32 bounds by casting u32 result into s32 when u32
15303	 * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded.
15304	 */
15305	if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) {
15306		dst_reg->s32_min_value = dst_reg->u32_min_value;
15307		dst_reg->s32_max_value = dst_reg->u32_max_value;
15308	} else {
15309		dst_reg->s32_min_value = S32_MIN;
15310		dst_reg->s32_max_value = S32_MAX;
15311	}
15312}
15313
15314static void scalar_min_max_xor(struct bpf_reg_state *dst_reg,
15315			       struct bpf_reg_state *src_reg)
15316{
15317	bool src_known = tnum_is_const(src_reg->var_off);
15318	bool dst_known = tnum_is_const(dst_reg->var_off);
15319
15320	if (src_known && dst_known) {
15321		/* dst_reg->var_off.value has been updated earlier */
15322		__mark_reg_known(dst_reg, dst_reg->var_off.value);
15323		return;
15324	}
15325
15326	/* We get both minimum and maximum from the var_off. */
15327	dst_reg->umin_value = dst_reg->var_off.value;
15328	dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask;
15329
15330	/* Safe to set s64 bounds by casting u64 result into s64 when u64
15331	 * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded.
15332	 */
15333	if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) {
15334		dst_reg->smin_value = dst_reg->umin_value;
15335		dst_reg->smax_value = dst_reg->umax_value;
15336	} else {
15337		dst_reg->smin_value = S64_MIN;
15338		dst_reg->smax_value = S64_MAX;
15339	}
15340
15341	__update_reg_bounds(dst_reg);
15342}
15343
15344static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
15345				   u64 umin_val, u64 umax_val)
15346{
15347	/* We lose all sign bit information (except what we can pick
15348	 * up from var_off)
15349	 */
15350	dst_reg->s32_min_value = S32_MIN;
15351	dst_reg->s32_max_value = S32_MAX;
15352	/* If we might shift our top bit out, then we know nothing */
15353	if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) {
15354		dst_reg->u32_min_value = 0;
15355		dst_reg->u32_max_value = U32_MAX;
15356	} else {
15357		dst_reg->u32_min_value <<= umin_val;
15358		dst_reg->u32_max_value <<= umax_val;
15359	}
15360}
15361
15362static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
15363				 struct bpf_reg_state *src_reg)
15364{
15365	u32 umax_val = src_reg->u32_max_value;
15366	u32 umin_val = src_reg->u32_min_value;
15367	/* u32 alu operation will zext upper bits */
15368	struct tnum subreg = tnum_subreg(dst_reg->var_off);
15369
15370	__scalar32_min_max_lsh(dst_reg, umin_val, umax_val);
15371	dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val));
15372	/* Not required but being careful mark reg64 bounds as unknown so
15373	 * that we are forced to pick them up from tnum and zext later and
15374	 * if some path skips this step we are still safe.
15375	 */
15376	__mark_reg64_unbounded(dst_reg);
15377	__update_reg32_bounds(dst_reg);
15378}
15379
15380static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg,
15381				   u64 umin_val, u64 umax_val)
15382{
15383	/* Special case <<32 because it is a common compiler pattern to sign
15384	 * extend subreg by doing <<32 s>>32. smin/smax assignments are correct
15385	 * because s32 bounds don't flip sign when shifting to the left by
15386	 * 32bits.
15387	 */
15388	if (umin_val == 32 && umax_val == 32) {
15389		dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32;
15390		dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32;
15391	} else {
15392		dst_reg->smax_value = S64_MAX;
15393		dst_reg->smin_value = S64_MIN;
15394	}
15395
15396	/* If we might shift our top bit out, then we know nothing */
15397	if (dst_reg->umax_value > 1ULL << (63 - umax_val)) {
15398		dst_reg->umin_value = 0;
15399		dst_reg->umax_value = U64_MAX;
15400	} else {
15401		dst_reg->umin_value <<= umin_val;
15402		dst_reg->umax_value <<= umax_val;
15403	}
15404}
15405
15406static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg,
15407			       struct bpf_reg_state *src_reg)
15408{
15409	u64 umax_val = src_reg->umax_value;
15410	u64 umin_val = src_reg->umin_value;
15411
15412	/* scalar64 calc uses 32bit unshifted bounds so must be called first */
15413	__scalar64_min_max_lsh(dst_reg, umin_val, umax_val);
15414	__scalar32_min_max_lsh(dst_reg, umin_val, umax_val);
15415
15416	dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val);
15417	/* We may learn something more from the var_off */
15418	__update_reg_bounds(dst_reg);
15419}
15420
15421static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg,
15422				 struct bpf_reg_state *src_reg)
15423{
15424	struct tnum subreg = tnum_subreg(dst_reg->var_off);
15425	u32 umax_val = src_reg->u32_max_value;
15426	u32 umin_val = src_reg->u32_min_value;
15427
15428	/* BPF_RSH is an unsigned shift.  If the value in dst_reg might
15429	 * be negative, then either:
15430	 * 1) src_reg might be zero, so the sign bit of the result is
15431	 *    unknown, so we lose our signed bounds
15432	 * 2) it's known negative, thus the unsigned bounds capture the
15433	 *    signed bounds
15434	 * 3) the signed bounds cross zero, so they tell us nothing
15435	 *    about the result
15436	 * If the value in dst_reg is known nonnegative, then again the
15437	 * unsigned bounds capture the signed bounds.
15438	 * Thus, in all cases it suffices to blow away our signed bounds
15439	 * and rely on inferring new ones from the unsigned bounds and
15440	 * var_off of the result.
15441	 */
15442	dst_reg->s32_min_value = S32_MIN;
15443	dst_reg->s32_max_value = S32_MAX;
15444
15445	dst_reg->var_off = tnum_rshift(subreg, umin_val);
15446	dst_reg->u32_min_value >>= umax_val;
15447	dst_reg->u32_max_value >>= umin_val;
15448
15449	__mark_reg64_unbounded(dst_reg);
15450	__update_reg32_bounds(dst_reg);
15451}
15452
15453static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg,
15454			       struct bpf_reg_state *src_reg)
15455{
15456	u64 umax_val = src_reg->umax_value;
15457	u64 umin_val = src_reg->umin_value;
15458
15459	/* BPF_RSH is an unsigned shift.  If the value in dst_reg might
15460	 * be negative, then either:
15461	 * 1) src_reg might be zero, so the sign bit of the result is
15462	 *    unknown, so we lose our signed bounds
15463	 * 2) it's known negative, thus the unsigned bounds capture the
15464	 *    signed bounds
15465	 * 3) the signed bounds cross zero, so they tell us nothing
15466	 *    about the result
15467	 * If the value in dst_reg is known nonnegative, then again the
15468	 * unsigned bounds capture the signed bounds.
15469	 * Thus, in all cases it suffices to blow away our signed bounds
15470	 * and rely on inferring new ones from the unsigned bounds and
15471	 * var_off of the result.
15472	 */
15473	dst_reg->smin_value = S64_MIN;
15474	dst_reg->smax_value = S64_MAX;
15475	dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val);
15476	dst_reg->umin_value >>= umax_val;
15477	dst_reg->umax_value >>= umin_val;
15478
15479	/* Its not easy to operate on alu32 bounds here because it depends
15480	 * on bits being shifted in. Take easy way out and mark unbounded
15481	 * so we can recalculate later from tnum.
15482	 */
15483	__mark_reg32_unbounded(dst_reg);
15484	__update_reg_bounds(dst_reg);
15485}
15486
15487static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg,
15488				  struct bpf_reg_state *src_reg)
15489{
15490	u64 umin_val = src_reg->u32_min_value;
15491
15492	/* Upon reaching here, src_known is true and
15493	 * umax_val is equal to umin_val.
15494	 */
15495	dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val);
15496	dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val);
15497
15498	dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32);
15499
15500	/* blow away the dst_reg umin_value/umax_value and rely on
15501	 * dst_reg var_off to refine the result.
15502	 */
15503	dst_reg->u32_min_value = 0;
15504	dst_reg->u32_max_value = U32_MAX;
15505
15506	__mark_reg64_unbounded(dst_reg);
15507	__update_reg32_bounds(dst_reg);
15508}
15509
15510static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg,
15511				struct bpf_reg_state *src_reg)
15512{
15513	u64 umin_val = src_reg->umin_value;
15514
15515	/* Upon reaching here, src_known is true and umax_val is equal
15516	 * to umin_val.
15517	 */
15518	dst_reg->smin_value >>= umin_val;
15519	dst_reg->smax_value >>= umin_val;
15520
15521	dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64);
15522
15523	/* blow away the dst_reg umin_value/umax_value and rely on
15524	 * dst_reg var_off to refine the result.
15525	 */
15526	dst_reg->umin_value = 0;
15527	dst_reg->umax_value = U64_MAX;
15528
15529	/* Its not easy to operate on alu32 bounds here because it depends
15530	 * on bits being shifted in from upper 32-bits. Take easy way out
15531	 * and mark unbounded so we can recalculate later from tnum.
15532	 */
15533	__mark_reg32_unbounded(dst_reg);
15534	__update_reg_bounds(dst_reg);
15535}
15536
15537static void scalar_byte_swap(struct bpf_reg_state *dst_reg, struct bpf_insn *insn)
15538{
15539	/*
15540	 * Byte swap operation - update var_off using tnum_bswap.
15541	 * Three cases:
15542	 * 1. bswap(16|32|64): opcode=0xd7 (BPF_END | BPF_ALU64 | BPF_TO_LE)
15543	 *    unconditional swap
15544	 * 2. to_le(16|32|64): opcode=0xd4 (BPF_END | BPF_ALU | BPF_TO_LE)
15545	 *    swap on big-endian, truncation or no-op on little-endian
15546	 * 3. to_be(16|32|64): opcode=0xdc (BPF_END | BPF_ALU | BPF_TO_BE)
15547	 *    swap on little-endian, truncation or no-op on big-endian
15548	 */
15549
15550	bool alu64 = BPF_CLASS(insn->code) == BPF_ALU64;
15551	bool to_le = BPF_SRC(insn->code) == BPF_TO_LE;
15552	bool is_big_endian;
15553#ifdef CONFIG_CPU_BIG_ENDIAN
15554	is_big_endian = true;
15555#else
15556	is_big_endian = false;
15557#endif
15558	/* Apply bswap if alu64 or switch between big-endian and little-endian machines */
15559	bool need_bswap = alu64 || (to_le == is_big_endian);
15560
15561	/*
15562	 * If the register is mutated, manually reset its scalar ID to break
15563	 * any existing ties and avoid incorrect bounds propagation.
15564	 */
15565	if (need_bswap || insn->imm == 16 || insn->imm == 32)
15566		dst_reg->id = 0;
15567
15568	if (need_bswap) {
15569		if (insn->imm == 16)
15570			dst_reg->var_off = tnum_bswap16(dst_reg->var_off);
15571		else if (insn->imm == 32)
15572			dst_reg->var_off = tnum_bswap32(dst_reg->var_off);
15573		else if (insn->imm == 64)
15574			dst_reg->var_off = tnum_bswap64(dst_reg->var_off);
15575		/*
15576		 * Byteswap scrambles the range, so we must reset bounds.
15577		 * Bounds will be re-derived from the new tnum later.
15578		 */
15579		__mark_reg_unbounded(dst_reg);
15580	}
15581	/* For bswap16/32, truncate dst register to match the swapped size */
15582	if (insn->imm == 16 || insn->imm == 32)
15583		coerce_reg_to_size(dst_reg, insn->imm / 8);
15584}
15585
15586static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn,
15587					     const struct bpf_reg_state *src_reg)
15588{
15589	bool src_is_const = false;
15590	u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32;
15591
15592	if (insn_bitness == 32) {
15593		if (tnum_subreg_is_const(src_reg->var_off)
15594		    && src_reg->s32_min_value == src_reg->s32_max_value
15595		    && src_reg->u32_min_value == src_reg->u32_max_value)
15596			src_is_const = true;
15597	} else {
15598		if (tnum_is_const(src_reg->var_off)
15599		    && src_reg->smin_value == src_reg->smax_value
15600		    && src_reg->umin_value == src_reg->umax_value)
15601			src_is_const = true;
15602	}
15603
15604	switch (BPF_OP(insn->code)) {
15605	case BPF_ADD:
15606	case BPF_SUB:
15607	case BPF_NEG:
15608	case BPF_AND:
15609	case BPF_XOR:
15610	case BPF_OR:
15611	case BPF_MUL:
15612	case BPF_END:
15613		return true;
15614
15615	/* Shift operators range is only computable if shift dimension operand
15616	 * is a constant. Shifts greater than 31 or 63 are undefined. This
15617	 * includes shifts by a negative number.
15618	 */
15619	case BPF_LSH:
15620	case BPF_RSH:
15621	case BPF_ARSH:
15622		return (src_is_const && src_reg->umax_value < insn_bitness);
15623	default:
15624		return false;
15625	}
15626}
15627
15628static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *insn,
15629			      struct bpf_reg_state *dst_reg)
15630{
15631	struct bpf_verifier_state *branch;
15632	struct bpf_reg_state *regs;
15633	bool alu32;
15634
15635	if (dst_reg->smin_value == -1 && dst_reg->smax_value == 0)
15636		alu32 = false;
15637	else if (dst_reg->s32_min_value == -1 && dst_reg->s32_max_value == 0)
15638		alu32 = true;
15639	else
15640		return 0;
15641
15642	branch = push_stack(env, env->insn_idx, env->insn_idx, false);
15643	if (IS_ERR(branch))
15644		return PTR_ERR(branch);
15645
15646	regs = branch->frame[branch->curframe]->regs;
15647	if (alu32) {
15648		__mark_reg32_known(&regs[insn->dst_reg], 0);
15649		__mark_reg32_known(dst_reg, -1ull);
15650	} else {
15651		__mark_reg_known(&regs[insn->dst_reg], 0);
15652		__mark_reg_known(dst_reg, -1ull);
15653	}
15654	return 0;
15655}
15656
15657/* WARNING: This function does calculations on 64-bit values, but the actual
15658 * execution may occur on 32-bit values. Therefore, things like bitshifts
15659 * need extra checks in the 32-bit case.
15660 */
15661static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
15662				      struct bpf_insn *insn,
15663				      struct bpf_reg_state *dst_reg,
15664				      struct bpf_reg_state src_reg)
15665{
15666	u8 opcode = BPF_OP(insn->code);
15667	bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64);
15668	int ret;
15669
15670	if (!is_safe_to_compute_dst_reg_range(insn, &src_reg)) {
15671		__mark_reg_unknown(env, dst_reg);
15672		return 0;
15673	}
15674
15675	if (sanitize_needed(opcode)) {
15676		ret = sanitize_val_alu(env, insn);
15677		if (ret < 0)
15678			return sanitize_err(env, insn, ret, NULL, NULL);
15679	}
15680
15681	/* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops.
15682	 * There are two classes of instructions: The first class we track both
15683	 * alu32 and alu64 sign/unsigned bounds independently this provides the
15684	 * greatest amount of precision when alu operations are mixed with jmp32
15685	 * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD,
15686	 * and BPF_OR. This is possible because these ops have fairly easy to
15687	 * understand and calculate behavior in both 32-bit and 64-bit alu ops.
15688	 * See alu32 verifier tests for examples. The second class of
15689	 * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy
15690	 * with regards to tracking sign/unsigned bounds because the bits may
15691	 * cross subreg boundaries in the alu64 case. When this happens we mark
15692	 * the reg unbounded in the subreg bound space and use the resulting
15693	 * tnum to calculate an approximation of the sign/unsigned bounds.
15694	 */
15695	switch (opcode) {
15696	case BPF_ADD:
15697		scalar32_min_max_add(dst_reg, &src_reg);
15698		scalar_min_max_add(dst_reg, &src_reg);
15699		dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off);
15700		break;
15701	case BPF_SUB:
15702		scalar32_min_max_sub(dst_reg, &src_reg);
15703		scalar_min_max_sub(dst_reg, &src_reg);
15704		dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off);
15705		break;
15706	case BPF_NEG:
15707		env->fake_reg[0] = *dst_reg;
15708		__mark_reg_known(dst_reg, 0);
15709		scalar32_min_max_sub(dst_reg, &env->fake_reg[0]);
15710		scalar_min_max_sub(dst_reg, &env->fake_reg[0]);
15711		dst_reg->var_off = tnum_neg(env->fake_reg[0].var_off);
15712		break;
15713	case BPF_MUL:
15714		dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off);
15715		scalar32_min_max_mul(dst_reg, &src_reg);
15716		scalar_min_max_mul(dst_reg, &src_reg);
15717		break;
15718	case BPF_AND:
15719		if (tnum_is_const(src_reg.var_off)) {
15720			ret = maybe_fork_scalars(env, insn, dst_reg);
15721			if (ret)
15722				return ret;
15723		}
15724		dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off);
15725		scalar32_min_max_and(dst_reg, &src_reg);
15726		scalar_min_max_and(dst_reg, &src_reg);
15727		break;
15728	case BPF_OR:
15729		if (tnum_is_const(src_reg.var_off)) {
15730			ret = maybe_fork_scalars(env, insn, dst_reg);
15731			if (ret)
15732				return ret;
15733		}
15734		dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off);
15735		scalar32_min_max_or(dst_reg, &src_reg);
15736		scalar_min_max_or(dst_reg, &src_reg);
15737		break;
15738	case BPF_XOR:
15739		dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off);
15740		scalar32_min_max_xor(dst_reg, &src_reg);
15741		scalar_min_max_xor(dst_reg, &src_reg);
15742		break;
15743	case BPF_LSH:
15744		if (alu32)
15745			scalar32_min_max_lsh(dst_reg, &src_reg);
15746		else
15747			scalar_min_max_lsh(dst_reg, &src_reg);
15748		break;
15749	case BPF_RSH:
15750		if (alu32)
15751			scalar32_min_max_rsh(dst_reg, &src_reg);
15752		else
15753			scalar_min_max_rsh(dst_reg, &src_reg);
15754		break;
15755	case BPF_ARSH:
15756		if (alu32)
15757			scalar32_min_max_arsh(dst_reg, &src_reg);
15758		else
15759			scalar_min_max_arsh(dst_reg, &src_reg);
15760		break;
15761	case BPF_END:
15762		scalar_byte_swap(dst_reg, insn);
15763		break;
15764	default:
15765		break;
15766	}
15767
15768	/*
15769	 * ALU32 ops are zero extended into 64bit register.
15770	 *
15771	 * BPF_END is already handled inside the helper (truncation),
15772	 * so skip zext here to avoid unexpected zero extension.
15773	 * e.g., le64: opcode=(BPF_END|BPF_ALU|BPF_TO_LE), imm=0x40
15774	 * This is a 64bit byte swap operation with alu32==true,
15775	 * but we should not zero extend the result.
15776	 */
15777	if (alu32 && opcode != BPF_END)
15778		zext_32_to_64(dst_reg);
15779	reg_bounds_sync(dst_reg);
15780	return 0;
15781}
15782
15783/* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max
15784 * and var_off.
15785 */
15786static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
15787				   struct bpf_insn *insn)
15788{
15789	struct bpf_verifier_state *vstate = env->cur_state;
15790	struct bpf_func_state *state = vstate->frame[vstate->curframe];
15791	struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;
15792	struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};
15793	bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64);
15794	u8 opcode = BPF_OP(insn->code);
15795	int err;
15796
15797	dst_reg = &regs[insn->dst_reg];
15798	src_reg = NULL;
15799
15800	if (dst_reg->type == PTR_TO_ARENA) {
15801		struct bpf_insn_aux_data *aux = cur_aux(env);
15802
15803		if (BPF_CLASS(insn->code) == BPF_ALU64)
15804			/*
15805			 * 32-bit operations zero upper bits automatically.
15806			 * 64-bit operations need to be converted to 32.
15807			 */
15808			aux->needs_zext = true;
15809
15810		/* Any arithmetic operations are allowed on arena pointers */
15811		return 0;
15812	}
15813
15814	if (dst_reg->type != SCALAR_VALUE)
15815		ptr_reg = dst_reg;
15816
15817	if (BPF_SRC(insn->code) == BPF_X) {
15818		src_reg = &regs[insn->src_reg];
15819		if (src_reg->type != SCALAR_VALUE) {
15820			if (dst_reg->type != SCALAR_VALUE) {
15821				/* Combining two pointers by any ALU op yields
15822				 * an arbitrary scalar. Disallow all math except
15823				 * pointer subtraction
15824				 */
15825				if (opcode == BPF_SUB && env->allow_ptr_leaks) {
15826					mark_reg_unknown(env, regs, insn->dst_reg);
15827					return 0;
15828				}
15829				verbose(env, "R%d pointer %s pointer prohibited\n",
15830					insn->dst_reg,
15831					bpf_alu_string[opcode >> 4]);
15832				return -EACCES;
15833			} else {
15834				/* scalar += pointer
15835				 * This is legal, but we have to reverse our
15836				 * src/dest handling in computing the range
15837				 */
15838				err = mark_chain_precision(env, insn->dst_reg);
15839				if (err)
15840					return err;
15841				return adjust_ptr_min_max_vals(env, insn,
15842							       src_reg, dst_reg);
15843			}
15844		} else if (ptr_reg) {
15845			/* pointer += scalar */
15846			err = mark_chain_precision(env, insn->src_reg);
15847			if (err)
15848				return err;
15849			return adjust_ptr_min_max_vals(env, insn,
15850						       dst_reg, src_reg);
15851		} else if (dst_reg->precise) {
15852			/* if dst_reg is precise, src_reg should be precise as well */
15853			err = mark_chain_precision(env, insn->src_reg);
15854			if (err)
15855				return err;
15856		}
15857	} else {
15858		/* Pretend the src is a reg with a known value, since we only
15859		 * need to be able to read from this state.
15860		 */
15861		off_reg.type = SCALAR_VALUE;
15862		__mark_reg_known(&off_reg, insn->imm);
15863		src_reg = &off_reg;
15864		if (ptr_reg) /* pointer += K */
15865			return adjust_ptr_min_max_vals(env, insn,
15866						       ptr_reg, src_reg);
15867	}
15868
15869	/* Got here implies adding two SCALAR_VALUEs */
15870	if (WARN_ON_ONCE(ptr_reg)) {
15871		print_verifier_state(env, vstate, vstate->curframe, true);
15872		verbose(env, "verifier internal error: unexpected ptr_reg\n");
15873		return -EFAULT;
15874	}
15875	if (WARN_ON(!src_reg)) {
15876		print_verifier_state(env, vstate, vstate->curframe, true);
15877		verbose(env, "verifier internal error: no src_reg\n");
15878		return -EFAULT;
15879	}
15880	err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);
15881	if (err)
15882		return err;
15883	/*
15884	 * Compilers can generate the code
15885	 * r1 = r2
15886	 * r1 += 0x1
15887	 * if r2 < 1000 goto ...
15888	 * use r1 in memory access
15889	 * So for 64-bit alu remember constant delta between r2 and r1 and
15890	 * update r1 after 'if' condition.
15891	 */
15892	if (env->bpf_capable &&
15893	    BPF_OP(insn->code) == BPF_ADD && !alu32 &&
15894	    dst_reg->id && is_reg_const(src_reg, false)) {
15895		u64 val = reg_const_value(src_reg, false);
15896
15897		if ((dst_reg->id & BPF_ADD_CONST) ||
15898		    /* prevent overflow in sync_linked_regs() later */
15899		    val > (u32)S32_MAX) {
15900			/*
15901			 * If the register already went through rX += val
15902			 * we cannot accumulate another val into rx->off.
15903			 */
15904			dst_reg->off = 0;
15905			dst_reg->id = 0;
15906		} else {
15907			dst_reg->id |= BPF_ADD_CONST;
15908			dst_reg->off = val;
15909		}
15910	} else {
15911		/*
15912		 * Make sure ID is cleared otherwise dst_reg min/max could be
15913		 * incorrectly propagated into other registers by sync_linked_regs()
15914		 */
15915		dst_reg->id = 0;
15916	}
15917	return 0;
15918}
15919
15920/* check validity of 32-bit and 64-bit arithmetic operations */
15921static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
15922{
15923	struct bpf_reg_state *regs = cur_regs(env);
15924	u8 opcode = BPF_OP(insn->code);
15925	int err;
15926
15927	if (opcode == BPF_END || opcode == BPF_NEG) {
15928		if (opcode == BPF_NEG) {
15929			if (BPF_SRC(insn->code) != BPF_K ||
15930			    insn->src_reg != BPF_REG_0 ||
15931			    insn->off != 0 || insn->imm != 0) {
15932				verbose(env, "BPF_NEG uses reserved fields\n");
15933				return -EINVAL;
15934			}
15935		} else {
15936			if (insn->src_reg != BPF_REG_0 || insn->off != 0 ||
15937			    (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) ||
15938			    (BPF_CLASS(insn->code) == BPF_ALU64 &&
15939			     BPF_SRC(insn->code) != BPF_TO_LE)) {
15940				verbose(env, "BPF_END uses reserved fields\n");
15941				return -EINVAL;
15942			}
15943		}
15944
15945		/* check src operand */
15946		err = check_reg_arg(env, insn->dst_reg, SRC_OP);
15947		if (err)
15948			return err;
15949
15950		if (is_pointer_value(env, insn->dst_reg)) {
15951			verbose(env, "R%d pointer arithmetic prohibited\n",
15952				insn->dst_reg);
15953			return -EACCES;
15954		}
15955
15956		/* check dest operand */
15957		if ((opcode == BPF_NEG || opcode == BPF_END) &&
15958		    regs[insn->dst_reg].type == SCALAR_VALUE) {
15959			err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
15960			err = err ?: adjust_scalar_min_max_vals(env, insn,
15961							 &regs[insn->dst_reg],
15962							 regs[insn->dst_reg]);
15963		} else {
15964			err = check_reg_arg(env, insn->dst_reg, DST_OP);
15965		}
15966		if (err)
15967			return err;
15968
15969	} else if (opcode == BPF_MOV) {
15970
15971		if (BPF_SRC(insn->code) == BPF_X) {
15972			if (BPF_CLASS(insn->code) == BPF_ALU) {
15973				if ((insn->off != 0 && insn->off != 8 && insn->off != 16) ||
15974				    insn->imm) {
15975					verbose(env, "BPF_MOV uses reserved fields\n");
15976					return -EINVAL;
15977				}
15978			} else if (insn->off == BPF_ADDR_SPACE_CAST) {
15979				if (insn->imm != 1 && insn->imm != 1u << 16) {
15980					verbose(env, "addr_space_cast insn can only convert between address space 1 and 0\n");
15981					return -EINVAL;
15982				}
15983				if (!env->prog->aux->arena) {
15984					verbose(env, "addr_space_cast insn can only be used in a program that has an associated arena\n");
15985					return -EINVAL;
15986				}
15987			} else {
15988				if ((insn->off != 0 && insn->off != 8 && insn->off != 16 &&
15989				     insn->off != 32) || insn->imm) {
15990					verbose(env, "BPF_MOV uses reserved fields\n");
15991					return -EINVAL;
15992				}
15993			}
15994
15995			/* check src operand */
15996			err = check_reg_arg(env, insn->src_reg, SRC_OP);
15997			if (err)
15998				return err;
15999		} else {
16000			if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
16001				verbose(env, "BPF_MOV uses reserved fields\n");
16002				return -EINVAL;
16003			}
16004		}
16005
16006		/* check dest operand, mark as required later */
16007		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
16008		if (err)
16009			return err;
16010
16011		if (BPF_SRC(insn->code) == BPF_X) {
16012			struct bpf_reg_state *src_reg = regs + insn->src_reg;
16013			struct bpf_reg_state *dst_reg = regs + insn->dst_reg;
16014
16015			if (BPF_CLASS(insn->code) == BPF_ALU64) {
16016				if (insn->imm) {
16017					/* off == BPF_ADDR_SPACE_CAST */
16018					mark_reg_unknown(env, regs, insn->dst_reg);
16019					if (insn->imm == 1) { /* cast from as(1) to as(0) */
16020						dst_reg->type = PTR_TO_ARENA;
16021						/* PTR_TO_ARENA is 32-bit */
16022						dst_reg->subreg_def = env->insn_idx + 1;
16023					}
16024				} else if (insn->off == 0) {
16025					/* case: R1 = R2
16026					 * copy register state to dest reg
16027					 */
16028					assign_scalar_id_before_mov(env, src_reg);
16029					copy_register_state(dst_reg, src_reg);
16030					dst_reg->subreg_def = DEF_NOT_SUBREG;
16031				} else {
16032					/* case: R1 = (s8, s16 s32)R2 */
16033					if (is_pointer_value(env, insn->src_reg)) {
16034						verbose(env,
16035							"R%d sign-extension part of pointer\n",
16036							insn->src_reg);
16037						return -EACCES;
16038					} else if (src_reg->type == SCALAR_VALUE) {
16039						bool no_sext;
16040
16041						no_sext = src_reg->umax_value < (1ULL << (insn->off - 1));
16042						if (no_sext)
16043							assign_scalar_id_before_mov(env, src_reg);
16044						copy_register_state(dst_reg, src_reg);
16045						if (!no_sext)
16046							dst_reg->id = 0;
16047						coerce_reg_to_size_sx(dst_reg, insn->off >> 3);
16048						dst_reg->subreg_def = DEF_NOT_SUBREG;
16049					} else {
16050						mark_reg_unknown(env, regs, insn->dst_reg);
16051					}
16052				}
16053			} else {
16054				/* R1 = (u32) R2 */
16055				if (is_pointer_value(env, insn->src_reg)) {
16056					verbose(env,
16057						"R%d partial copy of pointer\n",
16058						insn->src_reg);
16059					return -EACCES;
16060				} else if (src_reg->type == SCALAR_VALUE) {
16061					if (insn->off == 0) {
16062						bool is_src_reg_u32 = get_reg_width(src_reg) <= 32;
16063
16064						if (is_src_reg_u32)
16065							assign_scalar_id_before_mov(env, src_reg);
16066						copy_register_state(dst_reg, src_reg);
16067						/* Make sure ID is cleared if src_reg is not in u32
16068						 * range otherwise dst_reg min/max could be incorrectly
16069						 * propagated into src_reg by sync_linked_regs()
16070						 */
16071						if (!is_src_reg_u32)
16072							dst_reg->id = 0;
16073						dst_reg->subreg_def = env->insn_idx + 1;
16074					} else {
16075						/* case: W1 = (s8, s16)W2 */
16076						bool no_sext = src_reg->umax_value < (1ULL << (insn->off - 1));
16077
16078						if (no_sext)
16079							assign_scalar_id_before_mov(env, src_reg);
16080						copy_register_state(dst_reg, src_reg);
16081						if (!no_sext)
16082							dst_reg->id = 0;
16083						dst_reg->subreg_def = env->insn_idx + 1;
16084						coerce_subreg_to_size_sx(dst_reg, insn->off >> 3);
16085					}
16086				} else {
16087					mark_reg_unknown(env, regs,
16088							 insn->dst_reg);
16089				}
16090				zext_32_to_64(dst_reg);
16091				reg_bounds_sync(dst_reg);
16092			}
16093		} else {
16094			/* case: R = imm
16095			 * remember the value we stored into this reg
16096			 */
16097			/* clear any state __mark_reg_known doesn't set */
16098			mark_reg_unknown(env, regs, insn->dst_reg);
16099			regs[insn->dst_reg].type = SCALAR_VALUE;
16100			if (BPF_CLASS(insn->code) == BPF_ALU64) {
16101				__mark_reg_known(regs + insn->dst_reg,
16102						 insn->imm);
16103			} else {
16104				__mark_reg_known(regs + insn->dst_reg,
16105						 (u32)insn->imm);
16106			}
16107		}
16108
16109	} else if (opcode > BPF_END) {
16110		verbose(env, "invalid BPF_ALU opcode %x\n", opcode);
16111		return -EINVAL;
16112
16113	} else {	/* all other ALU ops: and, sub, xor, add, ... */
16114
16115		if (BPF_SRC(insn->code) == BPF_X) {
16116			if (insn->imm != 0 || (insn->off != 0 && insn->off != 1) ||
16117			    (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
16118				verbose(env, "BPF_ALU uses reserved fields\n");
16119				return -EINVAL;
16120			}
16121			/* check src1 operand */
16122			err = check_reg_arg(env, insn->src_reg, SRC_OP);
16123			if (err)
16124				return err;
16125		} else {
16126			if (insn->src_reg != BPF_REG_0 || (insn->off != 0 && insn->off != 1) ||
16127			    (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
16128				verbose(env, "BPF_ALU uses reserved fields\n");
16129				return -EINVAL;
16130			}
16131		}
16132
16133		/* check src2 operand */
16134		err = check_reg_arg(env, insn->dst_reg, SRC_OP);
16135		if (err)
16136			return err;
16137
16138		if ((opcode == BPF_MOD || opcode == BPF_DIV) &&
16139		    BPF_SRC(insn->code) == BPF_K && insn->imm == 0) {
16140			verbose(env, "div by zero\n");
16141			return -EINVAL;
16142		}
16143
16144		if ((opcode == BPF_LSH || opcode == BPF_RSH ||
16145		     opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {
16146			int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;
16147
16148			if (insn->imm < 0 || insn->imm >= size) {
16149				verbose(env, "invalid shift %d\n", insn->imm);
16150				return -EINVAL;
16151			}
16152		}
16153
16154		/* check dest operand */
16155		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
16156		err = err ?: adjust_reg_min_max_vals(env, insn);
16157		if (err)
16158			return err;
16159	}
16160
16161	return reg_bounds_sanity_check(env, &regs[insn->dst_reg], "alu");
16162}
16163
16164static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
16165				   struct bpf_reg_state *dst_reg,
16166				   enum bpf_reg_type type,
16167				   bool range_right_open)
16168{
16169	struct bpf_func_state *state;
16170	struct bpf_reg_state *reg;
16171	int new_range;
16172
16173	if (dst_reg->off < 0 ||
16174	    (dst_reg->off == 0 && range_right_open))
16175		/* This doesn't give us any range */
16176		return;
16177
16178	if (dst_reg->umax_value > MAX_PACKET_OFF ||
16179	    dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF)
16180		/* Risk of overflow.  For instance, ptr + (1<<63) may be less
16181		 * than pkt_end, but that's because it's also less than pkt.
16182		 */
16183		return;
16184
16185	new_range = dst_reg->off;
16186	if (range_right_open)
16187		new_range++;
16188
16189	/* Examples for register markings:
16190	 *
16191	 * pkt_data in dst register:
16192	 *
16193	 *   r2 = r3;
16194	 *   r2 += 8;
16195	 *   if (r2 > pkt_end) goto <handle exception>
16196	 *   <access okay>
16197	 *
16198	 *   r2 = r3;
16199	 *   r2 += 8;
16200	 *   if (r2 < pkt_end) goto <access okay>
16201	 *   <handle exception>
16202	 *
16203	 *   Where:
16204	 *     r2 == dst_reg, pkt_end == src_reg
16205	 *     r2=pkt(id=n,off=8,r=0)
16206	 *     r3=pkt(id=n,off=0,r=0)
16207	 *
16208	 * pkt_data in src register:
16209	 *
16210	 *   r2 = r3;
16211	 *   r2 += 8;
16212	 *   if (pkt_end >= r2) goto <access okay>
16213	 *   <handle exception>
16214	 *
16215	 *   r2 = r3;
16216	 *   r2 += 8;
16217	 *   if (pkt_end <= r2) goto <handle exception>
16218	 *   <access okay>
16219	 *
16220	 *   Where:
16221	 *     pkt_end == dst_reg, r2 == src_reg
16222	 *     r2=pkt(id=n,off=8,r=0)
16223	 *     r3=pkt(id=n,off=0,r=0)
16224	 *
16225	 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8)
16226	 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8)
16227	 * and [r3, r3 + 8-1) respectively is safe to access depending on
16228	 * the check.
16229	 */
16230
16231	/* If our ids match, then we must have the same max_value.  And we
16232	 * don't care about the other reg's fixed offset, since if it's too big
16233	 * the range won't allow anything.
16234	 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16.
16235	 */
16236	bpf_for_each_reg_in_vstate(vstate, state, reg, ({
16237		if (reg->type == type && reg->id == dst_reg->id)
16238			/* keep the maximum range already checked */
16239			reg->range = max(reg->range, new_range);
16240	}));
16241}
16242
16243/*
16244 * <reg1> <op> <reg2>, currently assuming reg2 is a constant
16245 */
16246static int is_scalar_branch_taken(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2,
16247				  u8 opcode, bool is_jmp32)
16248{
16249	struct tnum t1 = is_jmp32 ? tnum_subreg(reg1->var_off) : reg1->var_off;
16250	struct tnum t2 = is_jmp32 ? tnum_subreg(reg2->var_off) : reg2->var_off;
16251	u64 umin1 = is_jmp32 ? (u64)reg1->u32_min_value : reg1->umin_value;
16252	u64 umax1 = is_jmp32 ? (u64)reg1->u32_max_value : reg1->umax_value;
16253	s64 smin1 = is_jmp32 ? (s64)reg1->s32_min_value : reg1->smin_value;
16254	s64 smax1 = is_jmp32 ? (s64)reg1->s32_max_value : reg1->smax_value;
16255	u64 umin2 = is_jmp32 ? (u64)reg2->u32_min_value : reg2->umin_value;
16256	u64 umax2 = is_jmp32 ? (u64)reg2->u32_max_value : reg2->umax_value;
16257	s64 smin2 = is_jmp32 ? (s64)reg2->s32_min_value : reg2->smin_value;
16258	s64 smax2 = is_jmp32 ? (s64)reg2->s32_max_value : reg2->smax_value;
16259
16260	if (reg1 == reg2) {
16261		switch (opcode) {
16262		case BPF_JGE:
16263		case BPF_JLE:
16264		case BPF_JSGE:
16265		case BPF_JSLE:
16266		case BPF_JEQ:
16267			return 1;
16268		case BPF_JGT:
16269		case BPF_JLT:
16270		case BPF_JSGT:
16271		case BPF_JSLT:
16272		case BPF_JNE:
16273			return 0;
16274		case BPF_JSET:
16275			if (tnum_is_const(t1))
16276				return t1.value != 0;
16277			else
16278				return (smin1 <= 0 && smax1 >= 0) ? -1 : 1;
16279		default:
16280			return -1;
16281		}
16282	}
16283
16284	switch (opcode) {
16285	case BPF_JEQ:
16286		/* constants, umin/umax and smin/smax checks would be
16287		 * redundant in this case because they all should match
16288		 */
16289		if (tnum_is_const(t1) && tnum_is_const(t2))
16290			return t1.value == t2.value;
16291		if (!tnum_overlap(t1, t2))
16292			return 0;
16293		/* non-overlapping ranges */
16294		if (umin1 > umax2 || umax1 < umin2)
16295			return 0;
16296		if (smin1 > smax2 || smax1 < smin2)
16297			return 0;
16298		if (!is_jmp32) {
16299			/* if 64-bit ranges are inconclusive, see if we can
16300			 * utilize 32-bit subrange knowledge to eliminate
16301			 * branches that can't be taken a priori
16302			 */
16303			if (reg1->u32_min_value > reg2->u32_max_value ||
16304			    reg1->u32_max_value < reg2->u32_min_value)
16305				return 0;
16306			if (reg1->s32_min_value > reg2->s32_max_value ||
16307			    reg1->s32_max_value < reg2->s32_min_value)
16308				return 0;
16309		}
16310		break;
16311	case BPF_JNE:
16312		/* constants, umin/umax and smin/smax checks would be
16313		 * redundant in this case because they all should match
16314		 */
16315		if (tnum_is_const(t1) && tnum_is_const(t2))
16316			return t1.value != t2.value;
16317		if (!tnum_overlap(t1, t2))
16318			return 1;
16319		/* non-overlapping ranges */
16320		if (umin1 > umax2 || umax1 < umin2)
16321			return 1;
16322		if (smin1 > smax2 || smax1 < smin2)
16323			return 1;
16324		if (!is_jmp32) {
16325			/* if 64-bit ranges are inconclusive, see if we can
16326			 * utilize 32-bit subrange knowledge to eliminate
16327			 * branches that can't be taken a priori
16328			 */
16329			if (reg1->u32_min_value > reg2->u32_max_value ||
16330			    reg1->u32_max_value < reg2->u32_min_value)
16331				return 1;
16332			if (reg1->s32_min_value > reg2->s32_max_value ||
16333			    reg1->s32_max_value < reg2->s32_min_value)
16334				return 1;
16335		}
16336		break;
16337	case BPF_JSET:
16338		if (!is_reg_const(reg2, is_jmp32)) {
16339			swap(reg1, reg2);
16340			swap(t1, t2);
16341		}
16342		if (!is_reg_const(reg2, is_jmp32))
16343			return -1;
16344		if ((~t1.mask & t1.value) & t2.value)
16345			return 1;
16346		if (!((t1.mask | t1.value) & t2.value))
16347			return 0;
16348		break;
16349	case BPF_JGT:
16350		if (umin1 > umax2)
16351			return 1;
16352		else if (umax1 <= umin2)
16353			return 0;
16354		break;
16355	case BPF_JSGT:
16356		if (smin1 > smax2)
16357			return 1;
16358		else if (smax1 <= smin2)
16359			return 0;
16360		break;
16361	case BPF_JLT:
16362		if (umax1 < umin2)
16363			return 1;
16364		else if (umin1 >= umax2)
16365			return 0;
16366		break;
16367	case BPF_JSLT:
16368		if (smax1 < smin2)
16369			return 1;
16370		else if (smin1 >= smax2)
16371			return 0;
16372		break;
16373	case BPF_JGE:
16374		if (umin1 >= umax2)
16375			return 1;
16376		else if (umax1 < umin2)
16377			return 0;
16378		break;
16379	case BPF_JSGE:
16380		if (smin1 >= smax2)
16381			return 1;
16382		else if (smax1 < smin2)
16383			return 0;
16384		break;
16385	case BPF_JLE:
16386		if (umax1 <= umin2)
16387			return 1;
16388		else if (umin1 > umax2)
16389			return 0;
16390		break;
16391	case BPF_JSLE:
16392		if (smax1 <= smin2)
16393			return 1;
16394		else if (smin1 > smax2)
16395			return 0;
16396		break;
16397	}
16398
16399	return -1;
16400}
16401
16402static int flip_opcode(u32 opcode)
16403{
16404	/* How can we transform "a <op> b" into "b <op> a"? */
16405	static const u8 opcode_flip[16] = {
16406		/* these stay the same */
16407		[BPF_JEQ  >> 4] = BPF_JEQ,
16408		[BPF_JNE  >> 4] = BPF_JNE,
16409		[BPF_JSET >> 4] = BPF_JSET,
16410		/* these swap "lesser" and "greater" (L and G in the opcodes) */
16411		[BPF_JGE  >> 4] = BPF_JLE,
16412		[BPF_JGT  >> 4] = BPF_JLT,
16413		[BPF_JLE  >> 4] = BPF_JGE,
16414		[BPF_JLT  >> 4] = BPF_JGT,
16415		[BPF_JSGE >> 4] = BPF_JSLE,
16416		[BPF_JSGT >> 4] = BPF_JSLT,
16417		[BPF_JSLE >> 4] = BPF_JSGE,
16418		[BPF_JSLT >> 4] = BPF_JSGT
16419	};
16420	return opcode_flip[opcode >> 4];
16421}
16422
16423static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg,
16424				   struct bpf_reg_state *src_reg,
16425				   u8 opcode)
16426{
16427	struct bpf_reg_state *pkt;
16428
16429	if (src_reg->type == PTR_TO_PACKET_END) {
16430		pkt = dst_reg;
16431	} else if (dst_reg->type == PTR_TO_PACKET_END) {
16432		pkt = src_reg;
16433		opcode = flip_opcode(opcode);
16434	} else {
16435		return -1;
16436	}
16437
16438	if (pkt->range >= 0)
16439		return -1;
16440
16441	switch (opcode) {
16442	case BPF_JLE:
16443		/* pkt <= pkt_end */
16444		fallthrough;
16445	case BPF_JGT:
16446		/* pkt > pkt_end */
16447		if (pkt->range == BEYOND_PKT_END)
16448			/* pkt has at last one extra byte beyond pkt_end */
16449			return opcode == BPF_JGT;
16450		break;
16451	case BPF_JLT:
16452		/* pkt < pkt_end */
16453		fallthrough;
16454	case BPF_JGE:
16455		/* pkt >= pkt_end */
16456		if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END)
16457			return opcode == BPF_JGE;
16458		break;
16459	}
16460	return -1;
16461}
16462
16463/* compute branch direction of the expression "if (<reg1> opcode <reg2>) goto target;"
16464 * and return:
16465 *  1 - branch will be taken and "goto target" will be executed
16466 *  0 - branch will not be taken and fall-through to next insn
16467 * -1 - unknown. Example: "if (reg1 < 5)" is unknown when register value
16468 *      range [0,10]
16469 */
16470static int is_branch_taken(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2,
16471			   u8 opcode, bool is_jmp32)
16472{
16473	if (reg_is_pkt_pointer_any(reg1) && reg_is_pkt_pointer_any(reg2) && !is_jmp32)
16474		return is_pkt_ptr_branch_taken(reg1, reg2, opcode);
16475
16476	if (__is_pointer_value(false, reg1) || __is_pointer_value(false, reg2)) {
16477		u64 val;
16478
16479		/* arrange that reg2 is a scalar, and reg1 is a pointer */
16480		if (!is_reg_const(reg2, is_jmp32)) {
16481			opcode = flip_opcode(opcode);
16482			swap(reg1, reg2);
16483		}
16484		/* and ensure that reg2 is a constant */
16485		if (!is_reg_const(reg2, is_jmp32))
16486			return -1;
16487
16488		if (!reg_not_null(reg1))
16489			return -1;
16490
16491		/* If pointer is valid tests against zero will fail so we can
16492		 * use this to direct branch taken.
16493		 */
16494		val = reg_const_value(reg2, is_jmp32);
16495		if (val != 0)
16496			return -1;
16497
16498		switch (opcode) {
16499		case BPF_JEQ:
16500			return 0;
16501		case BPF_JNE:
16502			return 1;
16503		default:
16504			return -1;
16505		}
16506	}
16507
16508	/* now deal with two scalars, but not necessarily constants */
16509	return is_scalar_branch_taken(reg1, reg2, opcode, is_jmp32);
16510}
16511
16512/* Opcode that corresponds to a *false* branch condition.
16513 * E.g., if r1 < r2, then reverse (false) condition is r1 >= r2
16514 */
16515static u8 rev_opcode(u8 opcode)
16516{
16517	switch (opcode) {
16518	case BPF_JEQ:		return BPF_JNE;
16519	case BPF_JNE:		return BPF_JEQ;
16520	/* JSET doesn't have it's reverse opcode in BPF, so add
16521	 * BPF_X flag to denote the reverse of that operation
16522	 */
16523	case BPF_JSET:		return BPF_JSET | BPF_X;
16524	case BPF_JSET | BPF_X:	return BPF_JSET;
16525	case BPF_JGE:		return BPF_JLT;
16526	case BPF_JGT:		return BPF_JLE;
16527	case BPF_JLE:		return BPF_JGT;
16528	case BPF_JLT:		return BPF_JGE;
16529	case BPF_JSGE:		return BPF_JSLT;
16530	case BPF_JSGT:		return BPF_JSLE;
16531	case BPF_JSLE:		return BPF_JSGT;
16532	case BPF_JSLT:		return BPF_JSGE;
16533	default:		return 0;
16534	}
16535}
16536
16537/* Refine range knowledge for <reg1> <op> <reg>2 conditional operation. */
16538static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2,
16539				u8 opcode, bool is_jmp32)
16540{
16541	struct tnum t;
16542	u64 val;
16543
16544	/* In case of GE/GT/SGE/JST, reuse LE/LT/SLE/SLT logic from below */
16545	switch (opcode) {
16546	case BPF_JGE:
16547	case BPF_JGT:
16548	case BPF_JSGE:
16549	case BPF_JSGT:
16550		opcode = flip_opcode(opcode);
16551		swap(reg1, reg2);
16552		break;
16553	default:
16554		break;
16555	}
16556
16557	switch (opcode) {
16558	case BPF_JEQ:
16559		if (is_jmp32) {
16560			reg1->u32_min_value = max(reg1->u32_min_value, reg2->u32_min_value);
16561			reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value);
16562			reg1->s32_min_value = max(reg1->s32_min_value, reg2->s32_min_value);
16563			reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value);
16564			reg2->u32_min_value = reg1->u32_min_value;
16565			reg2->u32_max_value = reg1->u32_max_value;
16566			reg2->s32_min_value = reg1->s32_min_value;
16567			reg2->s32_max_value = reg1->s32_max_value;
16568
16569			t = tnum_intersect(tnum_subreg(reg1->var_off), tnum_subreg(reg2->var_off));
16570			reg1->var_off = tnum_with_subreg(reg1->var_off, t);
16571			reg2->var_off = tnum_with_subreg(reg2->var_off, t);
16572		} else {
16573			reg1->umin_value = max(reg1->umin_value, reg2->umin_value);
16574			reg1->umax_value = min(reg1->umax_value, reg2->umax_value);
16575			reg1->smin_value = max(reg1->smin_value, reg2->smin_value);
16576			reg1->smax_value = min(reg1->smax_value, reg2->smax_value);
16577			reg2->umin_value = reg1->umin_value;
16578			reg2->umax_value = reg1->umax_value;
16579			reg2->smin_value = reg1->smin_value;
16580			reg2->smax_value = reg1->smax_value;
16581
16582			reg1->var_off = tnum_intersect(reg1->var_off, reg2->var_off);
16583			reg2->var_off = reg1->var_off;
16584		}
16585		break;
16586	case BPF_JNE:
16587		if (!is_reg_const(reg2, is_jmp32))
16588			swap(reg1, reg2);
16589		if (!is_reg_const(reg2, is_jmp32))
16590			break;
16591
16592		/* try to recompute the bound of reg1 if reg2 is a const and
16593		 * is exactly the edge of reg1.
16594		 */
16595		val = reg_const_value(reg2, is_jmp32);
16596		if (is_jmp32) {
16597			/* u32_min_value is not equal to 0xffffffff at this point,
16598			 * because otherwise u32_max_value is 0xffffffff as well,
16599			 * in such a case both reg1 and reg2 would be constants,
16600			 * jump would be predicted and reg_set_min_max() won't
16601			 * be called.
16602			 *
16603			 * Same reasoning works for all {u,s}{min,max}{32,64} cases
16604			 * below.
16605			 */
16606			if (reg1->u32_min_value == (u32)val)
16607				reg1->u32_min_value++;
16608			if (reg1->u32_max_value == (u32)val)
16609				reg1->u32_max_value--;
16610			if (reg1->s32_min_value == (s32)val)
16611				reg1->s32_min_value++;
16612			if (reg1->s32_max_value == (s32)val)
16613				reg1->s32_max_value--;
16614		} else {
16615			if (reg1->umin_value == (u64)val)
16616				reg1->umin_value++;
16617			if (reg1->umax_value == (u64)val)
16618				reg1->umax_value--;
16619			if (reg1->smin_value == (s64)val)
16620				reg1->smin_value++;
16621			if (reg1->smax_value == (s64)val)
16622				reg1->smax_value--;
16623		}
16624		break;
16625	case BPF_JSET:
16626		if (!is_reg_const(reg2, is_jmp32))
16627			swap(reg1, reg2);
16628		if (!is_reg_const(reg2, is_jmp32))
16629			break;
16630		val = reg_const_value(reg2, is_jmp32);
16631		/* BPF_JSET (i.e., TRUE branch, *not* BPF_JSET | BPF_X)
16632		 * requires single bit to learn something useful. E.g., if we
16633		 * know that `r1 & 0x3` is true, then which bits (0, 1, or both)
16634		 * are actually set? We can learn something definite only if
16635		 * it's a single-bit value to begin with.
16636		 *
16637		 * BPF_JSET | BPF_X (i.e., negation of BPF_JSET) doesn't have
16638		 * this restriction. I.e., !(r1 & 0x3) means neither bit 0 nor
16639		 * bit 1 is set, which we can readily use in adjustments.
16640		 */
16641		if (!is_power_of_2(val))
16642			break;
16643		if (is_jmp32) {
16644			t = tnum_or(tnum_subreg(reg1->var_off), tnum_const(val));
16645			reg1->var_off = tnum_with_subreg(reg1->var_off, t);
16646		} else {
16647			reg1->var_off = tnum_or(reg1->var_off, tnum_const(val));
16648		}
16649		break;
16650	case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see rev_opcode() */
16651		if (!is_reg_const(reg2, is_jmp32))
16652			swap(reg1, reg2);
16653		if (!is_reg_const(reg2, is_jmp32))
16654			break;
16655		val = reg_const_value(reg2, is_jmp32);
16656		/* Forget the ranges before narrowing tnums, to avoid invariant
16657		 * violations if we're on a dead branch.
16658		 */
16659		__mark_reg_unbounded(reg1);
16660		if (is_jmp32) {
16661			t = tnum_and(tnum_subreg(reg1->var_off), tnum_const(~val));
16662			reg1->var_off = tnum_with_subreg(reg1->var_off, t);
16663		} else {
16664			reg1->var_off = tnum_and(reg1->var_off, tnum_const(~val));
16665		}
16666		break;
16667	case BPF_JLE:
16668		if (is_jmp32) {
16669			reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value);
16670			reg2->u32_min_value = max(reg1->u32_min_value, reg2->u32_min_value);
16671		} else {
16672			reg1->umax_value = min(reg1->umax_value, reg2->umax_value);
16673			reg2->umin_value = max(reg1->umin_value, reg2->umin_value);
16674		}
16675		break;
16676	case BPF_JLT:
16677		if (is_jmp32) {
16678			reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value - 1);
16679			reg2->u32_min_value = max(reg1->u32_min_value + 1, reg2->u32_min_value);
16680		} else {
16681			reg1->umax_value = min(reg1->umax_value, reg2->umax_value - 1);
16682			reg2->umin_value = max(reg1->umin_value + 1, reg2->umin_value);
16683		}
16684		break;
16685	case BPF_JSLE:
16686		if (is_jmp32) {
16687			reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value);
16688			reg2->s32_min_value = max(reg1->s32_min_value, reg2->s32_min_value);
16689		} else {
16690			reg1->smax_value = min(reg1->smax_value, reg2->smax_value);
16691			reg2->smin_value = max(reg1->smin_value, reg2->smin_value);
16692		}
16693		break;
16694	case BPF_JSLT:
16695		if (is_jmp32) {
16696			reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value - 1);
16697			reg2->s32_min_value = max(reg1->s32_min_value + 1, reg2->s32_min_value);
16698		} else {
16699			reg1->smax_value = min(reg1->smax_value, reg2->smax_value - 1);
16700			reg2->smin_value = max(reg1->smin_value + 1, reg2->smin_value);
16701		}
16702		break;
16703	default:
16704		return;
16705	}
16706}
16707
16708/* Adjusts the register min/max values in the case that the dst_reg and
16709 * src_reg are both SCALAR_VALUE registers (or we are simply doing a BPF_K
16710 * check, in which case we have a fake SCALAR_VALUE representing insn->imm).
16711 * Technically we can do similar adjustments for pointers to the same object,
16712 * but we don't support that right now.
16713 */
16714static int reg_set_min_max(struct bpf_verifier_env *env,
16715			   struct bpf_reg_state *true_reg1,
16716			   struct bpf_reg_state *true_reg2,
16717			   struct bpf_reg_state *false_reg1,
16718			   struct bpf_reg_state *false_reg2,
16719			   u8 opcode, bool is_jmp32)
16720{
16721	int err;
16722
16723	/* If either register is a pointer, we can't learn anything about its
16724	 * variable offset from the compare (unless they were a pointer into
16725	 * the same object, but we don't bother with that).
16726	 */
16727	if (false_reg1->type != SCALAR_VALUE || false_reg2->type != SCALAR_VALUE)
16728		return 0;
16729
16730	/* We compute branch direction for same SCALAR_VALUE registers in
16731	 * is_scalar_branch_taken(). For unknown branch directions (e.g., BPF_JSET)
16732	 * on the same registers, we don't need to adjust the min/max values.
16733	 */
16734	if (false_reg1 == false_reg2)
16735		return 0;
16736
16737	/* fallthrough (FALSE) branch */
16738	regs_refine_cond_op(false_reg1, false_reg2, rev_opcode(opcode), is_jmp32);
16739	reg_bounds_sync(false_reg1);
16740	reg_bounds_sync(false_reg2);
16741
16742	/* jump (TRUE) branch */
16743	regs_refine_cond_op(true_reg1, true_reg2, opcode, is_jmp32);
16744	reg_bounds_sync(true_reg1);
16745	reg_bounds_sync(true_reg2);
16746
16747	err = reg_bounds_sanity_check(env, true_reg1, "true_reg1");
16748	err = err ?: reg_bounds_sanity_check(env, true_reg2, "true_reg2");
16749	err = err ?: reg_bounds_sanity_check(env, false_reg1, "false_reg1");
16750	err = err ?: reg_bounds_sanity_check(env, false_reg2, "false_reg2");
16751	return err;
16752}
16753
16754static void mark_ptr_or_null_reg(struct bpf_func_state *state,
16755				 struct bpf_reg_state *reg, u32 id,
16756				 bool is_null)
16757{
16758	if (type_may_be_null(reg->type) && reg->id == id &&
16759	    (is_rcu_reg(reg) || !WARN_ON_ONCE(!reg->id))) {
16760		/* Old offset (both fixed and variable parts) should have been
16761		 * known-zero, because we don't allow pointer arithmetic on
16762		 * pointers that might be NULL. If we see this happening, don't
16763		 * convert the register.
16764		 *
16765		 * But in some cases, some helpers that return local kptrs
16766		 * advance offset for the returned pointer. In those cases, it
16767		 * is fine to expect to see reg->off.
16768		 */
16769		if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || !tnum_equals_const(reg->var_off, 0)))
16770			return;
16771		if (!(type_is_ptr_alloc_obj(reg->type) || type_is_non_owning_ref(reg->type)) &&
16772		    WARN_ON_ONCE(reg->off))
16773			return;
16774
16775		if (is_null) {
16776			reg->type = SCALAR_VALUE;
16777			/* We don't need id and ref_obj_id from this point
16778			 * onwards anymore, thus we should better reset it,
16779			 * so that state pruning has chances to take effect.
16780			 */
16781			reg->id = 0;
16782			reg->ref_obj_id = 0;
16783
16784			return;
16785		}
16786
16787		mark_ptr_not_null_reg(reg);
16788
16789		if (!reg_may_point_to_spin_lock(reg)) {
16790			/* For not-NULL ptr, reg->ref_obj_id will be reset
16791			 * in release_reference().
16792			 *
16793			 * reg->id is still used by spin_lock ptr. Other
16794			 * than spin_lock ptr type, reg->id can be reset.
16795			 */
16796			reg->id = 0;
16797		}
16798	}
16799}
16800
16801/* The logic is similar to find_good_pkt_pointers(), both could eventually
16802 * be folded together at some point.
16803 */
16804static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno,
16805				  bool is_null)
16806{
16807	struct bpf_func_state *state = vstate->frame[vstate->curframe];
16808	struct bpf_reg_state *regs = state->regs, *reg;
16809	u32 ref_obj_id = regs[regno].ref_obj_id;
16810	u32 id = regs[regno].id;
16811
16812	if (ref_obj_id && ref_obj_id == id && is_null)
16813		/* regs[regno] is in the " == NULL" branch.
16814		 * No one could have freed the reference state before
16815		 * doing the NULL check.
16816		 */
16817		WARN_ON_ONCE(release_reference_nomark(vstate, id));
16818
16819	bpf_for_each_reg_in_vstate(vstate, state, reg, ({
16820		mark_ptr_or_null_reg(state, reg, id, is_null);
16821	}));
16822}
16823
16824static bool try_match_pkt_pointers(const struct bpf_insn *insn,
16825				   struct bpf_reg_state *dst_reg,
16826				   struct bpf_reg_state *src_reg,
16827				   struct bpf_verifier_state *this_branch,
16828				   struct bpf_verifier_state *other_branch)
16829{
16830	if (BPF_SRC(insn->code) != BPF_X)
16831		return false;
16832
16833	/* Pointers are always 64-bit. */
16834	if (BPF_CLASS(insn->code) == BPF_JMP32)
16835		return false;
16836
16837	switch (BPF_OP(insn->code)) {
16838	case BPF_JGT:
16839		if ((dst_reg->type == PTR_TO_PACKET &&
16840		     src_reg->type == PTR_TO_PACKET_END) ||
16841		    (dst_reg->type == PTR_TO_PACKET_META &&
16842		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
16843			/* pkt_data' > pkt_end, pkt_meta' > pkt_data */
16844			find_good_pkt_pointers(this_branch, dst_reg,
16845					       dst_reg->type, false);
16846			mark_pkt_end(other_branch, insn->dst_reg, true);
16847		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
16848			    src_reg->type == PTR_TO_PACKET) ||
16849			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
16850			    src_reg->type == PTR_TO_PACKET_META)) {
16851			/* pkt_end > pkt_data', pkt_data > pkt_meta' */
16852			find_good_pkt_pointers(other_branch, src_reg,
16853					       src_reg->type, true);
16854			mark_pkt_end(this_branch, insn->src_reg, false);
16855		} else {
16856			return false;
16857		}
16858		break;
16859	case BPF_JLT:
16860		if ((dst_reg->type == PTR_TO_PACKET &&
16861		     src_reg->type == PTR_TO_PACKET_END) ||
16862		    (dst_reg->type == PTR_TO_PACKET_META &&
16863		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
16864			/* pkt_data' < pkt_end, pkt_meta' < pkt_data */
16865			find_good_pkt_pointers(other_branch, dst_reg,
16866					       dst_reg->type, true);
16867			mark_pkt_end(this_branch, insn->dst_reg, false);
16868		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
16869			    src_reg->type == PTR_TO_PACKET) ||
16870			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
16871			    src_reg->type == PTR_TO_PACKET_META)) {
16872			/* pkt_end < pkt_data', pkt_data > pkt_meta' */
16873			find_good_pkt_pointers(this_branch, src_reg,
16874					       src_reg->type, false);
16875			mark_pkt_end(other_branch, insn->src_reg, true);
16876		} else {
16877			return false;
16878		}
16879		break;
16880	case BPF_JGE:
16881		if ((dst_reg->type == PTR_TO_PACKET &&
16882		     src_reg->type == PTR_TO_PACKET_END) ||
16883		    (dst_reg->type == PTR_TO_PACKET_META &&
16884		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
16885			/* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */
16886			find_good_pkt_pointers(this_branch, dst_reg,
16887					       dst_reg->type, true);
16888			mark_pkt_end(other_branch, insn->dst_reg, false);
16889		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
16890			    src_reg->type == PTR_TO_PACKET) ||
16891			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
16892			    src_reg->type == PTR_TO_PACKET_META)) {
16893			/* pkt_end >= pkt_data', pkt_data >= pkt_meta' */
16894			find_good_pkt_pointers(other_branch, src_reg,
16895					       src_reg->type, false);
16896			mark_pkt_end(this_branch, insn->src_reg, true);
16897		} else {
16898			return false;
16899		}
16900		break;
16901	case BPF_JLE:
16902		if ((dst_reg->type == PTR_TO_PACKET &&
16903		     src_reg->type == PTR_TO_PACKET_END) ||
16904		    (dst_reg->type == PTR_TO_PACKET_META &&
16905		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
16906			/* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */
16907			find_good_pkt_pointers(other_branch, dst_reg,
16908					       dst_reg->type, false);
16909			mark_pkt_end(this_branch, insn->dst_reg, true);
16910		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
16911			    src_reg->type == PTR_TO_PACKET) ||
16912			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
16913			    src_reg->type == PTR_TO_PACKET_META)) {
16914			/* pkt_end <= pkt_data', pkt_data <= pkt_meta' */
16915			find_good_pkt_pointers(this_branch, src_reg,
16916					       src_reg->type, true);
16917			mark_pkt_end(other_branch, insn->src_reg, false);
16918		} else {
16919			return false;
16920		}
16921		break;
16922	default:
16923		return false;
16924	}
16925
16926	return true;
16927}
16928
16929static void __collect_linked_regs(struct linked_regs *reg_set, struct bpf_reg_state *reg,
16930				  u32 id, u32 frameno, u32 spi_or_reg, bool is_reg)
16931{
16932	struct linked_reg *e;
16933
16934	if (reg->type != SCALAR_VALUE || (reg->id & ~BPF_ADD_CONST) != id)
16935		return;
16936
16937	e = linked_regs_push(reg_set);
16938	if (e) {
16939		e->frameno = frameno;
16940		e->is_reg = is_reg;
16941		e->regno = spi_or_reg;
16942	} else {
16943		reg->id = 0;
16944	}
16945}
16946
16947/* For all R being scalar registers or spilled scalar registers
16948 * in verifier state, save R in linked_regs if R->id == id.
16949 * If there are too many Rs sharing same id, reset id for leftover Rs.
16950 */
16951static void collect_linked_regs(struct bpf_verifier_env *env,
16952				struct bpf_verifier_state *vstate,
16953				u32 id,
16954				struct linked_regs *linked_regs)
16955{
16956	struct bpf_insn_aux_data *aux = env->insn_aux_data;
16957	struct bpf_func_state *func;
16958	struct bpf_reg_state *reg;
16959	u16 live_regs;
16960	int i, j;
16961
16962	id = id & ~BPF_ADD_CONST;
16963	for (i = vstate->curframe; i >= 0; i--) {
16964		live_regs = aux[frame_insn_idx(vstate, i)].live_regs_before;
16965		func = vstate->frame[i];
16966		for (j = 0; j < BPF_REG_FP; j++) {
16967			if (!(live_regs & BIT(j)))
16968				continue;
16969			reg = &func->regs[j];
16970			__collect_linked_regs(linked_regs, reg, id, i, j, true);
16971		}
16972		for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
16973			if (!is_spilled_reg(&func->stack[j]))
16974				continue;
16975			reg = &func->stack[j].spilled_ptr;
16976			__collect_linked_regs(linked_regs, reg, id, i, j, false);
16977		}
16978	}
16979}
16980
16981/* For all R in linked_regs, copy known_reg range into R
16982 * if R->id == known_reg->id.
16983 */
16984static void sync_linked_regs(struct bpf_verifier_state *vstate, struct bpf_reg_state *known_reg,
16985			     struct linked_regs *linked_regs)
16986{
16987	struct bpf_reg_state fake_reg;
16988	struct bpf_reg_state *reg;
16989	struct linked_reg *e;
16990	int i;
16991
16992	for (i = 0; i < linked_regs->cnt; ++i) {
16993		e = &linked_regs->entries[i];
16994		reg = e->is_reg ? &vstate->frame[e->frameno]->regs[e->regno]
16995				: &vstate->frame[e->frameno]->stack[e->spi].spilled_ptr;
16996		if (reg->type != SCALAR_VALUE || reg == known_reg)
16997			continue;
16998		if ((reg->id & ~BPF_ADD_CONST) != (known_reg->id & ~BPF_ADD_CONST))
16999			continue;
17000		if ((!(reg->id & BPF_ADD_CONST) && !(known_reg->id & BPF_ADD_CONST)) ||
17001		    reg->off == known_reg->off) {
17002			s32 saved_subreg_def = reg->subreg_def;
17003
17004			copy_register_state(reg, known_reg);
17005			reg->subreg_def = saved_subreg_def;
17006		} else {
17007			s32 saved_subreg_def = reg->subreg_def;
17008			s32 saved_off = reg->off;
17009			u32 saved_id = reg->id;
17010
17011			fake_reg.type = SCALAR_VALUE;
17012			__mark_reg_known(&fake_reg, (s32)reg->off - (s32)known_reg->off);
17013
17014			/* reg = known_reg; reg += delta */
17015			copy_register_state(reg, known_reg);
17016			/*
17017			 * Must preserve off, id and subreg_def flag,
17018			 * otherwise another sync_linked_regs() will be incorrect.
17019			 */
17020			reg->off = saved_off;
17021			reg->id = saved_id;
17022			reg->subreg_def = saved_subreg_def;
17023
17024			scalar32_min_max_add(reg, &fake_reg);
17025			scalar_min_max_add(reg, &fake_reg);
17026			reg->var_off = tnum_add(reg->var_off, fake_reg.var_off);
17027		}
17028	}
17029}
17030
17031static int check_cond_jmp_op(struct bpf_verifier_env *env,
17032			     struct bpf_insn *insn, int *insn_idx)
17033{
17034	struct bpf_verifier_state *this_branch = env->cur_state;
17035	struct bpf_verifier_state *other_branch;
17036	struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;
17037	struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL;
17038	struct bpf_reg_state *eq_branch_regs;
17039	struct linked_regs linked_regs = {};
17040	u8 opcode = BPF_OP(insn->code);
17041	int insn_flags = 0;
17042	bool is_jmp32;
17043	int pred = -1;
17044	int err;
17045
17046	/* Only conditional jumps are expected to reach here. */
17047	if (opcode == BPF_JA || opcode > BPF_JCOND) {
17048		verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);
17049		return -EINVAL;
17050	}
17051
17052	if (opcode == BPF_JCOND) {
17053		struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;
17054		int idx = *insn_idx;
17055
17056		if (insn->code != (BPF_JMP | BPF_JCOND) ||
17057		    insn->src_reg != BPF_MAY_GOTO ||
17058		    insn->dst_reg || insn->imm) {
17059			verbose(env, "invalid may_goto imm %d\n", insn->imm);
17060			return -EINVAL;
17061		}
17062		prev_st = find_prev_entry(env, cur_st->parent, idx);
17063
17064		/* branch out 'fallthrough' insn as a new state to explore */
17065		queued_st = push_stack(env, idx + 1, idx, false);
17066		if (IS_ERR(queued_st))
17067			return PTR_ERR(queued_st);
17068
17069		queued_st->may_goto_depth++;
17070		if (prev_st)
17071			widen_imprecise_scalars(env, prev_st, queued_st);
17072		*insn_idx += insn->off;
17073		return 0;
17074	}
17075
17076	/* check src2 operand */
17077	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
17078	if (err)
17079		return err;
17080
17081	dst_reg = &regs[insn->dst_reg];
17082	if (BPF_SRC(insn->code) == BPF_X) {
17083		if (insn->imm != 0) {
17084			verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
17085			return -EINVAL;
17086		}
17087
17088		/* check src1 operand */
17089		err = check_reg_arg(env, insn->src_reg, SRC_OP);
17090		if (err)
17091			return err;
17092
17093		src_reg = &regs[insn->src_reg];
17094		if (!(reg_is_pkt_pointer_any(dst_reg) && reg_is_pkt_pointer_any(src_reg)) &&
17095		    is_pointer_value(env, insn->src_reg)) {
17096			verbose(env, "R%d pointer comparison prohibited\n",
17097				insn->src_reg);
17098			return -EACCES;
17099		}
17100
17101		if (src_reg->type == PTR_TO_STACK)
17102			insn_flags |= INSN_F_SRC_REG_STACK;
17103		if (dst_reg->type == PTR_TO_STACK)
17104			insn_flags |= INSN_F_DST_REG_STACK;
17105	} else {
17106		if (insn->src_reg != BPF_REG_0) {
17107			verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
17108			return -EINVAL;
17109		}
17110		src_reg = &env->fake_reg[0];
17111		memset(src_reg, 0, sizeof(*src_reg));
17112		src_reg->type = SCALAR_VALUE;
17113		__mark_reg_known(src_reg, insn->imm);
17114
17115		if (dst_reg->type == PTR_TO_STACK)
17116			insn_flags |= INSN_F_DST_REG_STACK;
17117	}
17118
17119	if (insn_flags) {
17120		err = push_jmp_history(env, this_branch, insn_flags, 0);
17121		if (err)
17122			return err;
17123	}
17124
17125	is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
17126	pred = is_branch_taken(dst_reg, src_reg, opcode, is_jmp32);
17127	if (pred >= 0) {
17128		/* If we get here with a dst_reg pointer type it is because
17129		 * above is_branch_taken() special cased the 0 comparison.
17130		 */
17131		if (!__is_pointer_value(false, dst_reg))
17132			err = mark_chain_precision(env, insn->dst_reg);
17133		if (BPF_SRC(insn->code) == BPF_X && !err &&
17134		    !__is_pointer_value(false, src_reg))
17135			err = mark_chain_precision(env, insn->src_reg);
17136		if (err)
17137			return err;
17138	}
17139
17140	if (pred == 1) {
17141		/* Only follow the goto, ignore fall-through. If needed, push
17142		 * the fall-through branch for simulation under speculative
17143		 * execution.
17144		 */
17145		if (!env->bypass_spec_v1) {
17146			err = sanitize_speculative_path(env, insn, *insn_idx + 1, *insn_idx);
17147			if (err < 0)
17148				return err;
17149		}
17150		if (env->log.level & BPF_LOG_LEVEL)
17151			print_insn_state(env, this_branch, this_branch->curframe);
17152		*insn_idx += insn->off;
17153		return 0;
17154	} else if (pred == 0) {
17155		/* Only follow the fall-through branch, since that's where the
17156		 * program will go. If needed, push the goto branch for
17157		 * simulation under speculative execution.
17158		 */
17159		if (!env->bypass_spec_v1) {
17160			err = sanitize_speculative_path(env, insn, *insn_idx + insn->off + 1,
17161							*insn_idx);
17162			if (err < 0)
17163				return err;
17164		}
17165		if (env->log.level & BPF_LOG_LEVEL)
17166			print_insn_state(env, this_branch, this_branch->curframe);
17167		return 0;
17168	}
17169
17170	/* Push scalar registers sharing same ID to jump history,
17171	 * do this before creating 'other_branch', so that both
17172	 * 'this_branch' and 'other_branch' share this history
17173	 * if parent state is created.
17174	 */
17175	if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id)
17176		collect_linked_regs(env, this_branch, src_reg->id, &linked_regs);
17177	if (dst_reg->type == SCALAR_VALUE && dst_reg->id)
17178		collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs);
17179	if (linked_regs.cnt > 1) {
17180		err = push_jmp_history(env, this_branch, 0, linked_regs_pack(&linked_regs));
17181		if (err)
17182			return err;
17183	}
17184
17185	other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, false);
17186	if (IS_ERR(other_branch))
17187		return PTR_ERR(other_branch);
17188	other_branch_regs = other_branch->frame[other_branch->curframe]->regs;
17189
17190	if (BPF_SRC(insn->code) == BPF_X) {
17191		err = reg_set_min_max(env,
17192				      &other_branch_regs[insn->dst_reg],
17193				      &other_branch_regs[insn->src_reg],
17194				      dst_reg, src_reg, opcode, is_jmp32);
17195	} else /* BPF_SRC(insn->code) == BPF_K */ {
17196		/* reg_set_min_max() can mangle the fake_reg. Make a copy
17197		 * so that these are two different memory locations. The
17198		 * src_reg is not used beyond here in context of K.
17199		 */
17200		memcpy(&env->fake_reg[1], &env->fake_reg[0],
17201		       sizeof(env->fake_reg[0]));
17202		err = reg_set_min_max(env,
17203				      &other_branch_regs[insn->dst_reg],
17204				      &env->fake_reg[0],
17205				      dst_reg, &env->fake_reg[1],
17206				      opcode, is_jmp32);
17207	}
17208	if (err)
17209		return err;
17210
17211	if (BPF_SRC(insn->code) == BPF_X &&
17212	    src_reg->type == SCALAR_VALUE && src_reg->id &&
17213	    !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) {
17214		sync_linked_regs(this_branch, src_reg, &linked_regs);
17215		sync_linked_regs(other_branch, &other_branch_regs[insn->src_reg], &linked_regs);
17216	}
17217	if (dst_reg->type == SCALAR_VALUE && dst_reg->id &&
17218	    !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) {
17219		sync_linked_regs(this_branch, dst_reg, &linked_regs);
17220		sync_linked_regs(other_branch, &other_branch_regs[insn->dst_reg], &linked_regs);
17221	}
17222
17223	/* if one pointer register is compared to another pointer
17224	 * register check if PTR_MAYBE_NULL could be lifted.
17225	 * E.g. register A - maybe null
17226	 *      register B - not null
17227	 * for JNE A, B, ... - A is not null in the false branch;
17228	 * for JEQ A, B, ... - A is not null in the true branch.
17229	 *
17230	 * Since PTR_TO_BTF_ID points to a kernel struct that does
17231	 * not need to be null checked by the BPF program, i.e.,
17232	 * could be null even without PTR_MAYBE_NULL marking, so
17233	 * only propagate nullness when neither reg is that type.
17234	 */
17235	if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X &&
17236	    __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) &&
17237	    type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type) &&
17238	    base_type(src_reg->type) != PTR_TO_BTF_ID &&
17239	    base_type(dst_reg->type) != PTR_TO_BTF_ID) {
17240		eq_branch_regs = NULL;
17241		switch (opcode) {
17242		case BPF_JEQ:
17243			eq_branch_regs = other_branch_regs;
17244			break;
17245		case BPF_JNE:
17246			eq_branch_regs = regs;
17247			break;
17248		default:
17249			/* do nothing */
17250			break;
17251		}
17252		if (eq_branch_regs) {
17253			if (type_may_be_null(src_reg->type))
17254				mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]);
17255			else
17256				mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]);
17257		}
17258	}
17259
17260	/* detect if R == 0 where R is returned from bpf_map_lookup_elem().
17261	 * NOTE: these optimizations below are related with pointer comparison
17262	 *       which will never be JMP32.
17263	 */
17264	if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&
17265	    insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) &&
17266	    type_may_be_null(dst_reg->type)) {
17267		/* Mark all identical registers in each branch as either
17268		 * safe or unknown depending R == 0 or R != 0 conditional.
17269		 */
17270		mark_ptr_or_null_regs(this_branch, insn->dst_reg,
17271				      opcode == BPF_JNE);
17272		mark_ptr_or_null_regs(other_branch, insn->dst_reg,
17273				      opcode == BPF_JEQ);
17274	} else if (!try_match_pkt_pointers(insn, dst_reg, &regs[insn->src_reg],
17275					   this_branch, other_branch) &&
17276		   is_pointer_value(env, insn->dst_reg)) {
17277		verbose(env, "R%d pointer comparison prohibited\n",
17278			insn->dst_reg);
17279		return -EACCES;
17280	}
17281	if (env->log.level & BPF_LOG_LEVEL)
17282		print_insn_state(env, this_branch, this_branch->curframe);
17283	return 0;
17284}
17285
17286/* verify BPF_LD_IMM64 instruction */
17287static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
17288{
17289	struct bpf_insn_aux_data *aux = cur_aux(env);
17290	struct bpf_reg_state *regs = cur_regs(env);
17291	struct bpf_reg_state *dst_reg;
17292	struct bpf_map *map;
17293	int err;
17294
17295	if (BPF_SIZE(insn->code) != BPF_DW) {
17296		verbose(env, "invalid BPF_LD_IMM insn\n");
17297		return -EINVAL;
17298	}
17299	if (insn->off != 0) {
17300		verbose(env, "BPF_LD_IMM64 uses reserved fields\n");
17301		return -EINVAL;
17302	}
17303
17304	err = check_reg_arg(env, insn->dst_reg, DST_OP);
17305	if (err)
17306		return err;
17307
17308	dst_reg = &regs[insn->dst_reg];
17309	if (insn->src_reg == 0) {
17310		u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
17311
17312		dst_reg->type = SCALAR_VALUE;
17313		__mark_reg_known(&regs[insn->dst_reg], imm);
17314		return 0;
17315	}
17316
17317	/* All special src_reg cases are listed below. From this point onwards
17318	 * we either succeed and assign a corresponding dst_reg->type after
17319	 * zeroing the offset, or fail and reject the program.
17320	 */
17321	mark_reg_known_zero(env, regs, insn->dst_reg);
17322
17323	if (insn->src_reg == BPF_PSEUDO_BTF_ID) {
17324		dst_reg->type = aux->btf_var.reg_type;
17325		switch (base_type(dst_reg->type)) {
17326		case PTR_TO_MEM:
17327			dst_reg->mem_size = aux->btf_var.mem_size;
17328			break;
17329		case PTR_TO_BTF_ID:
17330			dst_reg->btf = aux->btf_var.btf;
17331			dst_reg->btf_id = aux->btf_var.btf_id;
17332			break;
17333		default:
17334			verifier_bug(env, "pseudo btf id: unexpected dst reg type");
17335			return -EFAULT;
17336		}
17337		return 0;
17338	}
17339
17340	if (insn->src_reg == BPF_PSEUDO_FUNC) {
17341		struct bpf_prog_aux *aux = env->prog->aux;
17342		u32 subprogno = find_subprog(env,
17343					     env->insn_idx + insn->imm + 1);
17344
17345		if (!aux->func_info) {
17346			verbose(env, "missing btf func_info\n");
17347			return -EINVAL;
17348		}
17349		if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) {
17350			verbose(env, "callback function not static\n");
17351			return -EINVAL;
17352		}
17353
17354		dst_reg->type = PTR_TO_FUNC;
17355		dst_reg->subprogno = subprogno;
17356		return 0;
17357	}
17358
17359	map = env->used_maps[aux->map_index];
17360	dst_reg->map_ptr = map;
17361
17362	if (insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
17363	    insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) {
17364		if (map->map_type == BPF_MAP_TYPE_ARENA) {
17365			__mark_reg_unknown(env, dst_reg);
17366			return 0;
17367		}
17368		dst_reg->type = PTR_TO_MAP_VALUE;
17369		dst_reg->off = aux->map_off;
17370		WARN_ON_ONCE(map->map_type != BPF_MAP_TYPE_INSN_ARRAY &&
17371			     map->max_entries != 1);
17372		/* We want reg->id to be same (0) as map_value is not distinct */
17373	} else if (insn->src_reg == BPF_PSEUDO_MAP_FD ||
17374		   insn->src_reg == BPF_PSEUDO_MAP_IDX) {
17375		dst_reg->type = CONST_PTR_TO_MAP;
17376	} else {
17377		verifier_bug(env, "unexpected src reg value for ldimm64");
17378		return -EFAULT;
17379	}
17380
17381	return 0;
17382}
17383
17384static bool may_access_skb(enum bpf_prog_type type)
17385{
17386	switch (type) {
17387	case BPF_PROG_TYPE_SOCKET_FILTER:
17388	case BPF_PROG_TYPE_SCHED_CLS:
17389	case BPF_PROG_TYPE_SCHED_ACT:
17390		return true;
17391	default:
17392		return false;
17393	}
17394}
17395
17396/* verify safety of LD_ABS|LD_IND instructions:
17397 * - they can only appear in the programs where ctx == skb
17398 * - since they are wrappers of function calls, they scratch R1-R5 registers,
17399 *   preserve R6-R9, and store return value into R0
17400 *
17401 * Implicit input:
17402 *   ctx == skb == R6 == CTX
17403 *
17404 * Explicit input:
17405 *   SRC == any register
17406 *   IMM == 32-bit immediate
17407 *
17408 * Output:
17409 *   R0 - 8/16/32-bit skb data converted to cpu endianness
17410 */
17411static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
17412{
17413	struct bpf_reg_state *regs = cur_regs(env);
17414	static const int ctx_reg = BPF_REG_6;
17415	u8 mode = BPF_MODE(insn->code);
17416	int i, err;
17417
17418	if (!may_access_skb(resolve_prog_type(env->prog))) {
17419		verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");
17420		return -EINVAL;
17421	}
17422
17423	if (!env->ops->gen_ld_abs) {
17424		verifier_bug(env, "gen_ld_abs is null");
17425		return -EFAULT;
17426	}
17427
17428	if (insn->dst_reg != BPF_REG_0 || insn->off != 0 ||
17429	    BPF_SIZE(insn->code) == BPF_DW ||
17430	    (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) {
17431		verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");
17432		return -EINVAL;
17433	}
17434
17435	/* check whether implicit source operand (register R6) is readable */
17436	err = check_reg_arg(env, ctx_reg, SRC_OP);
17437	if (err)
17438		return err;
17439
17440	/* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as
17441	 * gen_ld_abs() may terminate the program at runtime, leading to
17442	 * reference leak.
17443	 */
17444	err = check_resource_leak(env, false, true, "BPF_LD_[ABS|IND]");
17445	if (err)
17446		return err;
17447
17448	if (regs[ctx_reg].type != PTR_TO_CTX) {
17449		verbose(env,
17450			"at the time of BPF_LD_ABS|IND R6 != pointer to skb\n");
17451		return -EINVAL;
17452	}
17453
17454	if (mode == BPF_IND) {
17455		/* check explicit source operand */
17456		err = check_reg_arg(env, insn->src_reg, SRC_OP);
17457		if (err)
17458			return err;
17459	}
17460
17461	err = check_ptr_off_reg(env, &regs[ctx_reg], ctx_reg);
17462	if (err < 0)
17463		return err;
17464
17465	/* reset caller saved regs to unreadable */
17466	for (i = 0; i < CALLER_SAVED_REGS; i++) {
17467		mark_reg_not_init(env, regs, caller_saved[i]);
17468		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
17469	}
17470
17471	/* mark destination R0 register as readable, since it contains
17472	 * the value fetched from the packet.
17473	 * Already marked as written above.
17474	 */
17475	mark_reg_unknown(env, regs, BPF_REG_0);
17476	/* ld_abs load up to 32-bit skb data. */
17477	regs[BPF_REG_0].subreg_def = env->insn_idx + 1;
17478	return 0;
17479}
17480
17481static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name)
17482{
17483	const char *exit_ctx = "At program exit";
17484	struct tnum enforce_attach_type_range = tnum_unknown;
17485	const struct bpf_prog *prog = env->prog;
17486	struct bpf_reg_state *reg = reg_state(env, regno);
17487	struct bpf_retval_range range = retval_range(0, 1);
17488	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
17489	int err;
17490	struct bpf_func_state *frame = env->cur_state->frame[0];
17491	const bool is_subprog = frame->subprogno;
17492	bool return_32bit = false;
17493	const struct btf_type *reg_type, *ret_type = NULL;
17494
17495	/* LSM and struct_ops func-ptr's return type could be "void" */
17496	if (!is_subprog || frame->in_exception_callback_fn) {
17497		switch (prog_type) {
17498		case BPF_PROG_TYPE_LSM:
17499			if (prog->expected_attach_type == BPF_LSM_CGROUP)
17500				/* See below, can be 0 or 0-1 depending on hook. */
17501				break;
17502			if (!prog->aux->attach_func_proto->type)
17503				return 0;
17504			break;
17505		case BPF_PROG_TYPE_STRUCT_OPS:
17506			if (!prog->aux->attach_func_proto->type)
17507				return 0;
17508
17509			if (frame->in_exception_callback_fn)
17510				break;
17511
17512			/* Allow a struct_ops program to return a referenced kptr if it
17513			 * matches the operator's return type and is in its unmodified
17514			 * form. A scalar zero (i.e., a null pointer) is also allowed.
17515			 */
17516			reg_type = reg->btf ? btf_type_by_id(reg->btf, reg->btf_id) : NULL;
17517			ret_type = btf_type_resolve_ptr(prog->aux->attach_btf,
17518							prog->aux->attach_func_proto->type,
17519							NULL);
17520			if (ret_type && ret_type == reg_type && reg->ref_obj_id)
17521				return __check_ptr_off_reg(env, reg, regno, false);
17522			break;
17523		default:
17524			break;
17525		}
17526	}
17527
17528	/* eBPF calling convention is such that R0 is used
17529	 * to return the value from eBPF program.
17530	 * Make sure that it's readable at this time
17531	 * of bpf_exit, which means that program wrote
17532	 * something into it earlier
17533	 */
17534	err = check_reg_arg(env, regno, SRC_OP);
17535	if (err)
17536		return err;
17537
17538	if (is_pointer_value(env, regno)) {
17539		verbose(env, "R%d leaks addr as return value\n", regno);
17540		return -EACCES;
17541	}
17542
17543	if (frame->in_async_callback_fn) {
17544		exit_ctx = "At async callback return";
17545		range = frame->callback_ret_range;
17546		goto enforce_retval;
17547	}
17548
17549	if (is_subprog && !frame->in_exception_callback_fn) {
17550		if (reg->type != SCALAR_VALUE) {
17551			verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n",
17552				regno, reg_type_str(env, reg->type));
17553			return -EINVAL;
17554		}
17555		return 0;
17556	}
17557
17558	switch (prog_type) {
17559	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
17560		if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG ||
17561		    env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG ||
17562		    env->prog->expected_attach_type == BPF_CGROUP_UNIX_RECVMSG ||
17563		    env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME ||
17564		    env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME ||
17565		    env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETPEERNAME ||
17566		    env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||
17567		    env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME ||
17568		    env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETSOCKNAME)
17569			range = retval_range(1, 1);
17570		if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND ||
17571		    env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND)
17572			range = retval_range(0, 3);
17573		break;
17574	case BPF_PROG_TYPE_CGROUP_SKB:
17575		if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {
17576			range = retval_range(0, 3);
17577			enforce_attach_type_range = tnum_range(2, 3);
17578		}
17579		break;
17580	case BPF_PROG_TYPE_CGROUP_SOCK:
17581	case BPF_PROG_TYPE_SOCK_OPS:
17582	case BPF_PROG_TYPE_CGROUP_DEVICE:
17583	case BPF_PROG_TYPE_CGROUP_SYSCTL:
17584	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
17585		break;
17586	case BPF_PROG_TYPE_RAW_TRACEPOINT:
17587		if (!env->prog->aux->attach_btf_id)
17588			return 0;
17589		range = retval_range(0, 0);
17590		break;
17591	case BPF_PROG_TYPE_TRACING:
17592		switch (env->prog->expected_attach_type) {
17593		case BPF_TRACE_FENTRY:
17594		case BPF_TRACE_FEXIT:
17595			range = retval_range(0, 0);
17596			break;
17597		case BPF_TRACE_RAW_TP:
17598		case BPF_MODIFY_RETURN:
17599			return 0;
17600		case BPF_TRACE_ITER:
17601			break;
17602		default:
17603			return -ENOTSUPP;
17604		}
17605		break;
17606	case BPF_PROG_TYPE_KPROBE:
17607		switch (env->prog->expected_attach_type) {
17608		case BPF_TRACE_KPROBE_SESSION:
17609		case BPF_TRACE_UPROBE_SESSION:
17610			range = retval_range(0, 1);
17611			break;
17612		default:
17613			return 0;
17614		}
17615		break;
17616	case BPF_PROG_TYPE_SK_LOOKUP:
17617		range = retval_range(SK_DROP, SK_PASS);
17618		break;
17619
17620	case BPF_PROG_TYPE_LSM:
17621		if (env->prog->expected_attach_type != BPF_LSM_CGROUP) {
17622			/* no range found, any return value is allowed */
17623			if (!get_func_retval_range(env->prog, &range))
17624				return 0;
17625			/* no restricted range, any return value is allowed */
17626			if (range.minval == S32_MIN && range.maxval == S32_MAX)
17627				return 0;
17628			return_32bit = true;
17629		} else if (!env->prog->aux->attach_func_proto->type) {
17630			/* Make sure programs that attach to void
17631			 * hooks don't try to modify return value.
17632			 */
17633			range = retval_range(1, 1);
17634		}
17635		break;
17636
17637	case BPF_PROG_TYPE_NETFILTER:
17638		range = retval_range(NF_DROP, NF_ACCEPT);
17639		break;
17640	case BPF_PROG_TYPE_STRUCT_OPS:
17641		if (!ret_type)
17642			return 0;
17643		range = retval_range(0, 0);
17644		break;
17645	case BPF_PROG_TYPE_EXT:
17646		/* freplace program can return anything as its return value
17647		 * depends on the to-be-replaced kernel func or bpf program.
17648		 */
17649	default:
17650		return 0;
17651	}
17652
17653enforce_retval:
17654	if (reg->type != SCALAR_VALUE) {
17655		verbose(env, "%s the register R%d is not a known value (%s)\n",
17656			exit_ctx, regno, reg_type_str(env, reg->type));
17657		return -EINVAL;
17658	}
17659
17660	err = mark_chain_precision(env, regno);
17661	if (err)
17662		return err;
17663
17664	if (!retval_range_within(range, reg, return_32bit)) {
17665		verbose_invalid_scalar(env, reg, range, exit_ctx, reg_name);
17666		if (!is_subprog &&
17667		    prog->expected_attach_type == BPF_LSM_CGROUP &&
17668		    prog_type == BPF_PROG_TYPE_LSM &&
17669		    !prog->aux->attach_func_proto->type)
17670			verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
17671		return -EINVAL;
17672	}
17673
17674	if (!tnum_is_unknown(enforce_attach_type_range) &&
17675	    tnum_in(enforce_attach_type_range, reg->var_off))
17676		env->prog->enforce_expected_attach_type = 1;
17677	return 0;
17678}
17679
17680static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off)
17681{
17682	struct bpf_subprog_info *subprog;
17683
17684	subprog = bpf_find_containing_subprog(env, off);
17685	subprog->changes_pkt_data = true;
17686}
17687
17688static void mark_subprog_might_sleep(struct bpf_verifier_env *env, int off)
17689{
17690	struct bpf_subprog_info *subprog;
17691
17692	subprog = bpf_find_containing_subprog(env, off);
17693	subprog->might_sleep = true;
17694}
17695
17696/* 't' is an index of a call-site.
17697 * 'w' is a callee entry point.
17698 * Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED.
17699 * Rely on DFS traversal order and absence of recursive calls to guarantee that
17700 * callee's change_pkt_data marks would be correct at that moment.
17701 */
17702static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w)
17703{
17704	struct bpf_subprog_info *caller, *callee;
17705
17706	caller = bpf_find_containing_subprog(env, t);
17707	callee = bpf_find_containing_subprog(env, w);
17708	caller->changes_pkt_data |= callee->changes_pkt_data;
17709	caller->might_sleep |= callee->might_sleep;
17710}
17711
17712/* non-recursive DFS pseudo code
17713 * 1  procedure DFS-iterative(G,v):
17714 * 2      label v as discovered
17715 * 3      let S be a stack
17716 * 4      S.push(v)
17717 * 5      while S is not empty
17718 * 6            t <- S.peek()
17719 * 7            if t is what we're looking for:
17720 * 8                return t
17721 * 9            for all edges e in G.adjacentEdges(t) do
17722 * 10               if edge e is already labelled
17723 * 11                   continue with the next edge
17724 * 12               w <- G.adjacentVertex(t,e)
17725 * 13               if vertex w is not discovered and not explored
17726 * 14                   label e as tree-edge
17727 * 15                   label w as discovered
17728 * 16                   S.push(w)
17729 * 17                   continue at 5
17730 * 18               else if vertex w is discovered
17731 * 19                   label e as back-edge
17732 * 20               else
17733 * 21                   // vertex w is explored
17734 * 22                   label e as forward- or cross-edge
17735 * 23           label t as explored
17736 * 24           S.pop()
17737 *
17738 * convention:
17739 * 0x10 - discovered
17740 * 0x11 - discovered and fall-through edge labelled
17741 * 0x12 - discovered and fall-through and branch edges labelled
17742 * 0x20 - explored
17743 */
17744
17745enum {
17746	DISCOVERED = 0x10,
17747	EXPLORED = 0x20,
17748	FALLTHROUGH = 1,
17749	BRANCH = 2,
17750};
17751
17752static void mark_prune_point(struct bpf_verifier_env *env, int idx)
17753{
17754	env->insn_aux_data[idx].prune_point = true;
17755}
17756
17757static bool is_prune_point(struct bpf_verifier_env *env, int insn_idx)
17758{
17759	return env->insn_aux_data[insn_idx].prune_point;
17760}
17761
17762static void mark_force_checkpoint(struct bpf_verifier_env *env, int idx)
17763{
17764	env->insn_aux_data[idx].force_checkpoint = true;
17765}
17766
17767static bool is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx)
17768{
17769	return env->insn_aux_data[insn_idx].force_checkpoint;
17770}
17771
17772static void mark_calls_callback(struct bpf_verifier_env *env, int idx)
17773{
17774	env->insn_aux_data[idx].calls_callback = true;
17775}
17776
17777bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx)
17778{
17779	return env->insn_aux_data[insn_idx].calls_callback;
17780}
17781
17782enum {
17783	DONE_EXPLORING = 0,
17784	KEEP_EXPLORING = 1,
17785};
17786
17787/* t, w, e - match pseudo-code above:
17788 * t - index of current instruction
17789 * w - next instruction
17790 * e - edge
17791 */
17792static int push_insn(int t, int w, int e, struct bpf_verifier_env *env)
17793{
17794	int *insn_stack = env->cfg.insn_stack;
17795	int *insn_state = env->cfg.insn_state;
17796
17797	if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH))
17798		return DONE_EXPLORING;
17799
17800	if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH))
17801		return DONE_EXPLORING;
17802
17803	if (w < 0 || w >= env->prog->len) {
17804		verbose_linfo(env, t, "%d: ", t);
17805		verbose(env, "jump out of range from insn %d to %d\n", t, w);
17806		return -EINVAL;
17807	}
17808
17809	if (e == BRANCH) {
17810		/* mark branch target for state pruning */
17811		mark_prune_point(env, w);
17812		mark_jmp_point(env, w);
17813	}
17814
17815	if (insn_state[w] == 0) {
17816		/* tree-edge */
17817		insn_state[t] = DISCOVERED | e;
17818		insn_state[w] = DISCOVERED;
17819		if (env->cfg.cur_stack >= env->prog->len)
17820			return -E2BIG;
17821		insn_stack[env->cfg.cur_stack++] = w;
17822		return KEEP_EXPLORING;
17823	} else if ((insn_state[w] & 0xF0) == DISCOVERED) {
17824		if (env->bpf_capable)
17825			return DONE_EXPLORING;
17826		verbose_linfo(env, t, "%d: ", t);
17827		verbose_linfo(env, w, "%d: ", w);
17828		verbose(env, "back-edge from insn %d to %d\n", t, w);
17829		return -EINVAL;
17830	} else if (insn_state[w] == EXPLORED) {
17831		/* forward- or cross-edge */
17832		insn_state[t] = DISCOVERED | e;
17833	} else {
17834		verifier_bug(env, "insn state internal bug");
17835		return -EFAULT;
17836	}
17837	return DONE_EXPLORING;
17838}
17839
17840static int visit_func_call_insn(int t, struct bpf_insn *insns,
17841				struct bpf_verifier_env *env,
17842				bool visit_callee)
17843{
17844	int ret, insn_sz;
17845	int w;
17846
17847	insn_sz = bpf_is_ldimm64(&insns[t]) ? 2 : 1;
17848	ret = push_insn(t, t + insn_sz, FALLTHROUGH, env);
17849	if (ret)
17850		return ret;
17851
17852	mark_prune_point(env, t + insn_sz);
17853	/* when we exit from subprog, we need to record non-linear history */
17854	mark_jmp_point(env, t + insn_sz);
17855
17856	if (visit_callee) {
17857		w = t + insns[t].imm + 1;
17858		mark_prune_point(env, t);
17859		merge_callee_effects(env, t, w);
17860		ret = push_insn(t, w, BRANCH, env);
17861	}
17862	return ret;
17863}
17864
17865/* Bitmask with 1s for all caller saved registers */
17866#define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1)
17867
17868/* True if do_misc_fixups() replaces calls to helper number 'imm',
17869 * replacement patch is presumed to follow bpf_fastcall contract
17870 * (see mark_fastcall_pattern_for_call() below).
17871 */
17872static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
17873{
17874	switch (imm) {
17875#ifdef CONFIG_X86_64
17876	case BPF_FUNC_get_smp_processor_id:
17877		return env->prog->jit_requested && bpf_jit_supports_percpu_insn();
17878#endif
17879	default:
17880		return false;
17881	}
17882}
17883
17884struct call_summary {
17885	u8 num_params;
17886	bool is_void;
17887	bool fastcall;
17888};
17889
17890/* If @call is a kfunc or helper call, fills @cs and returns true,
17891 * otherwise returns false.
17892 */
17893static bool get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call,
17894			     struct call_summary *cs)
17895{
17896	struct bpf_kfunc_call_arg_meta meta;
17897	const struct bpf_func_proto *fn;
17898	int i;
17899
17900	if (bpf_helper_call(call)) {
17901
17902		if (get_helper_proto(env, call->imm, &fn) < 0)
17903			/* error would be reported later */
17904			return false;
17905		cs->fastcall = fn->allow_fastcall &&
17906			       (verifier_inlines_helper_call(env, call->imm) ||
17907				bpf_jit_inlines_helper_call(call->imm));
17908		cs->is_void = fn->ret_type == RET_VOID;
17909		cs->num_params = 0;
17910		for (i = 0; i < ARRAY_SIZE(fn->arg_type); ++i) {
17911			if (fn->arg_type[i] == ARG_DONTCARE)
17912				break;
17913			cs->num_params++;
17914		}
17915		return true;
17916	}
17917
17918	if (bpf_pseudo_kfunc_call(call)) {
17919		int err;
17920
17921		err = fetch_kfunc_meta(env, call, &meta, NULL);
17922		if (err < 0)
17923			/* error would be reported later */
17924			return false;
17925		cs->num_params = btf_type_vlen(meta.func_proto);
17926		cs->fastcall = meta.kfunc_flags & KF_FASTCALL;
17927		cs->is_void = btf_type_is_void(btf_type_by_id(meta.btf, meta.func_proto->type));
17928		return true;
17929	}
17930
17931	return false;
17932}
17933
17934/* LLVM define a bpf_fastcall function attribute.
17935 * This attribute means that function scratches only some of
17936 * the caller saved registers defined by ABI.
17937 * For BPF the set of such registers could be defined as follows:
17938 * - R0 is scratched only if function is non-void;
17939 * - R1-R5 are scratched only if corresponding parameter type is defined
17940 *   in the function prototype.
17941 *
17942 * The contract between kernel and clang allows to simultaneously use
17943 * such functions and maintain backwards compatibility with old
17944 * kernels that don't understand bpf_fastcall calls:
17945 *
17946 * - for bpf_fastcall calls clang allocates registers as-if relevant r0-r5
17947 *   registers are not scratched by the call;
17948 *
17949 * - as a post-processing step, clang visits each bpf_fastcall call and adds
17950 *   spill/fill for every live r0-r5;
17951 *
17952 * - stack offsets used for the spill/fill are allocated as lowest
17953 *   stack offsets in whole function and are not used for any other
17954 *   purposes;
17955 *
17956 * - when kernel loads a program, it looks for such patterns
17957 *   (bpf_fastcall function surrounded by spills/fills) and checks if
17958 *   spill/fill stack offsets are used exclusively in fastcall patterns;
17959 *
17960 * - if so, and if verifier or current JIT inlines the call to the
17961 *   bpf_fastcall function (e.g. a helper call), kernel removes unnecessary
17962 *   spill/fill pairs;
17963 *
17964 * - when old kernel loads a program, presence of spill/fill pairs
17965 *   keeps BPF program valid, albeit slightly less efficient.
17966 *
17967 * For example:
17968 *
17969 *   r1 = 1;
17970 *   r2 = 2;
17971 *   *(u64 *)(r10 - 8)  = r1;            r1 = 1;
17972 *   *(u64 *)(r10 - 16) = r2;            r2 = 2;
17973 *   call %[to_be_inlined]         -->   call %[to_be_inlined]
17974 *   r2 = *(u64 *)(r10 - 16);            r0 = r1;
17975 *   r1 = *(u64 *)(r10 - 8);             r0 += r2;
17976 *   r0 = r1;                            exit;
17977 *   r0 += r2;
17978 *   exit;
17979 *
17980 * The purpose of mark_fastcall_pattern_for_call is to:
17981 * - look for such patterns;
17982 * - mark spill and fill instructions in env->insn_aux_data[*].fastcall_pattern;
17983 * - mark set env->insn_aux_data[*].fastcall_spills_num for call instruction;
17984 * - update env->subprog_info[*]->fastcall_stack_off to find an offset
17985 *   at which bpf_fastcall spill/fill stack slots start;
17986 * - update env->subprog_info[*]->keep_fastcall_stack.
17987 *
17988 * The .fastcall_pattern and .fastcall_stack_off are used by
17989 * check_fastcall_stack_contract() to check if every stack access to
17990 * fastcall spill/fill stack slot originates from spill/fill
17991 * instructions, members of fastcall patterns.
17992 *
17993 * If such condition holds true for a subprogram, fastcall patterns could
17994 * be rewritten by remove_fastcall_spills_fills().
17995 * Otherwise bpf_fastcall patterns are not changed in the subprogram
17996 * (code, presumably, generated by an older clang version).
17997 *
17998 * For example, it is *not* safe to remove spill/fill below:
17999 *
18000 *   r1 = 1;
18001 *   *(u64 *)(r10 - 8)  = r1;            r1 = 1;
18002 *   call %[to_be_inlined]         -->   call %[to_be_inlined]
18003 *   r1 = *(u64 *)(r10 - 8);             r0 = *(u64 *)(r10 - 8);  <---- wrong !!!
18004 *   r0 = *(u64 *)(r10 - 8);             r0 += r1;
18005 *   r0 += r1;                           exit;
18006 *   exit;
18007 */
18008static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,
18009					   struct bpf_subprog_info *subprog,
18010					   int insn_idx, s16 lowest_off)
18011{
18012	struct bpf_insn *insns = env->prog->insnsi, *stx, *ldx;
18013	struct bpf_insn *call = &env->prog->insnsi[insn_idx];
18014	u32 clobbered_regs_mask;
18015	struct call_summary cs;
18016	u32 expected_regs_mask;
18017	s16 off;
18018	int i;
18019
18020	if (!get_call_summary(env, call, &cs))
18021		return;
18022
18023	/* A bitmask specifying which caller saved registers are clobbered
18024	 * by a call to a helper/kfunc *as if* this helper/kfunc follows
18025	 * bpf_fastcall contract:
18026	 * - includes R0 if function is non-void;
18027	 * - includes R1-R5 if corresponding parameter has is described
18028	 *   in the function prototype.
18029	 */
18030	clobbered_regs_mask = GENMASK(cs.num_params, cs.is_void ? 1 : 0);
18031	/* e.g. if helper call clobbers r{0,1}, expect r{2,3,4,5} in the pattern */
18032	expected_regs_mask = ~clobbered_regs_mask & ALL_CALLER_SAVED_REGS;
18033
18034	/* match pairs of form:
18035	 *
18036	 * *(u64 *)(r10 - Y) = rX   (where Y % 8 == 0)
18037	 * ...
18038	 * call %[to_be_inlined]
18039	 * ...
18040	 * rX = *(u64 *)(r10 - Y)
18041	 */
18042	for (i = 1, off = lowest_off; i <= ARRAY_SIZE(caller_saved); ++i, off += BPF_REG_SIZE) {
18043		if (insn_idx - i < 0 || insn_idx + i >= env->prog->len)
18044			break;
18045		stx = &insns[insn_idx - i];
18046		ldx = &insns[insn_idx + i];
18047		/* must be a stack spill/fill pair */
18048		if (stx->code != (BPF_STX | BPF_MEM | BPF_DW) ||
18049		    ldx->code != (BPF_LDX | BPF_MEM | BPF_DW) ||
18050		    stx->dst_reg != BPF_REG_10 ||
18051		    ldx->src_reg != BPF_REG_10)
18052			break;
18053		/* must be a spill/fill for the same reg */
18054		if (stx->src_reg != ldx->dst_reg)
18055			break;
18056		/* must be one of the previously unseen registers */
18057		if ((BIT(stx->src_reg) & expected_regs_mask) == 0)
18058			break;
18059		/* must be a spill/fill for the same expected offset,
18060		 * no need to check offset alignment, BPF_DW stack access
18061		 * is always 8-byte aligned.
18062		 */
18063		if (stx->off != off || ldx->off != off)
18064			break;
18065		expected_regs_mask &= ~BIT(stx->src_reg);
18066		env->insn_aux_data[insn_idx - i].fastcall_pattern = 1;
18067		env->insn_aux_data[insn_idx + i].fastcall_pattern = 1;
18068	}
18069	if (i == 1)
18070		return;
18071
18072	/* Conditionally set 'fastcall_spills_num' to allow forward
18073	 * compatibility when more helper functions are marked as
18074	 * bpf_fastcall at compile time than current kernel supports, e.g:
18075	 *
18076	 *   1: *(u64 *)(r10 - 8) = r1
18077	 *   2: call A                  ;; assume A is bpf_fastcall for current kernel
18078	 *   3: r1 = *(u64 *)(r10 - 8)
18079	 *   4: *(u64 *)(r10 - 8) = r1
18080	 *   5: call B                  ;; assume B is not bpf_fastcall for current kernel
18081	 *   6: r1 = *(u64 *)(r10 - 8)
18082	 *
18083	 * There is no need to block bpf_fastcall rewrite for such program.
18084	 * Set 'fastcall_pattern' for both calls to keep check_fastcall_stack_contract() happy,
18085	 * don't set 'fastcall_spills_num' for call B so that remove_fastcall_spills_fills()
18086	 * does not remove spill/fill pair {4,6}.
18087	 */
18088	if (cs.fastcall)
18089		env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1;
18090	else
18091		subprog->keep_fastcall_stack = 1;
18092	subprog->fastcall_stack_off = min(subprog->fastcall_stack_off, off);
18093}
18094
18095static int mark_fastcall_patterns(struct bpf_verifier_env *env)
18096{
18097	struct bpf_subprog_info *subprog = env->subprog_info;
18098	struct bpf_insn *insn;
18099	s16 lowest_off;
18100	int s, i;
18101
18102	for (s = 0; s < env->subprog_cnt; ++s, ++subprog) {
18103		/* find lowest stack spill offset used in this subprog */
18104		lowest_off = 0;
18105		for (i = subprog->start; i < (subprog + 1)->start; ++i) {
18106			insn = env->prog->insnsi + i;
18107			if (insn->code != (BPF_STX | BPF_MEM | BPF_DW) ||
18108			    insn->dst_reg != BPF_REG_10)
18109				continue;
18110			lowest_off = min(lowest_off, insn->off);
18111		}
18112		/* use this offset to find fastcall patterns */
18113		for (i = subprog->start; i < (subprog + 1)->start; ++i) {
18114			insn = env->prog->insnsi + i;
18115			if (insn->code != (BPF_JMP | BPF_CALL))
18116				continue;
18117			mark_fastcall_pattern_for_call(env, subprog, i, lowest_off);
18118		}
18119	}
18120	return 0;
18121}
18122
18123static struct bpf_iarray *iarray_realloc(struct bpf_iarray *old, size_t n_elem)
18124{
18125	size_t new_size = sizeof(struct bpf_iarray) + n_elem * sizeof(old->items[0]);
18126	struct bpf_iarray *new;
18127
18128	new = kvrealloc(old, new_size, GFP_KERNEL_ACCOUNT);
18129	if (!new) {
18130		/* this is what callers always want, so simplify the call site */
18131		kvfree(old);
18132		return NULL;
18133	}
18134
18135	new->cnt = n_elem;
18136	return new;
18137}
18138
18139static int copy_insn_array(struct bpf_map *map, u32 start, u32 end, u32 *items)
18140{
18141	struct bpf_insn_array_value *value;
18142	u32 i;
18143
18144	for (i = start; i <= end; i++) {
18145		value = map->ops->map_lookup_elem(map, &i);
18146		/*
18147		 * map_lookup_elem of an array map will never return an error,
18148		 * but not checking it makes some static analysers to worry
18149		 */
18150		if (IS_ERR(value))
18151			return PTR_ERR(value);
18152		else if (!value)
18153			return -EINVAL;
18154		items[i - start] = value->xlated_off;
18155	}
18156	return 0;
18157}
18158
18159static int cmp_ptr_to_u32(const void *a, const void *b)
18160{
18161	return *(u32 *)a - *(u32 *)b;
18162}
18163
18164static int sort_insn_array_uniq(u32 *items, int cnt)
18165{
18166	int unique = 1;
18167	int i;
18168
18169	sort(items, cnt, sizeof(items[0]), cmp_ptr_to_u32, NULL);
18170
18171	for (i = 1; i < cnt; i++)
18172		if (items[i] != items[unique - 1])
18173			items[unique++] = items[i];
18174
18175	return unique;
18176}
18177
18178/*
18179 * sort_unique({map[start], ..., map[end]}) into off
18180 */
18181static int copy_insn_array_uniq(struct bpf_map *map, u32 start, u32 end, u32 *off)
18182{
18183	u32 n = end - start + 1;
18184	int err;
18185
18186	err = copy_insn_array(map, start, end, off);
18187	if (err)
18188		return err;
18189
18190	return sort_insn_array_uniq(off, n);
18191}
18192
18193/*
18194 * Copy all unique offsets from the map
18195 */
18196static struct bpf_iarray *jt_from_map(struct bpf_map *map)
18197{
18198	struct bpf_iarray *jt;
18199	int err;
18200	int n;
18201
18202	jt = iarray_realloc(NULL, map->max_entries);
18203	if (!jt)
18204		return ERR_PTR(-ENOMEM);
18205
18206	n = copy_insn_array_uniq(map, 0, map->max_entries - 1, jt->items);
18207	if (n < 0) {
18208		err = n;
18209		goto err_free;
18210	}
18211	if (n == 0) {
18212		err = -EINVAL;
18213		goto err_free;
18214	}
18215	jt->cnt = n;
18216	return jt;
18217
18218err_free:
18219	kvfree(jt);
18220	return ERR_PTR(err);
18221}
18222
18223/*
18224 * Find and collect all maps which fit in the subprog. Return the result as one
18225 * combined jump table in jt->items (allocated with kvcalloc)
18226 */
18227static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env,
18228					  int subprog_start, int subprog_end)
18229{
18230	struct bpf_iarray *jt = NULL;
18231	struct bpf_map *map;
18232	struct bpf_iarray *jt_cur;
18233	int i;
18234
18235	for (i = 0; i < env->insn_array_map_cnt; i++) {
18236		/*
18237		 * TODO (when needed): collect only jump tables, not static keys
18238		 * or maps for indirect calls
18239		 */
18240		map = env->insn_array_maps[i];
18241
18242		jt_cur = jt_from_map(map);
18243		if (IS_ERR(jt_cur)) {
18244			kvfree(jt);
18245			return jt_cur;
18246		}
18247
18248		/*
18249		 * This is enough to check one element. The full table is
18250		 * checked to fit inside the subprog later in create_jt()
18251		 */
18252		if (jt_cur->items[0] >= subprog_start && jt_cur->items[0] < subprog_end) {
18253			u32 old_cnt = jt ? jt->cnt : 0;
18254			jt = iarray_realloc(jt, old_cnt + jt_cur->cnt);
18255			if (!jt) {
18256				kvfree(jt_cur);
18257				return ERR_PTR(-ENOMEM);
18258			}
18259			memcpy(jt->items + old_cnt, jt_cur->items, jt_cur->cnt << 2);
18260		}
18261
18262		kvfree(jt_cur);
18263	}
18264
18265	if (!jt) {
18266		verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start);
18267		return ERR_PTR(-EINVAL);
18268	}
18269
18270	jt->cnt = sort_insn_array_uniq(jt->items, jt->cnt);
18271	return jt;
18272}
18273
18274static struct bpf_iarray *
18275create_jt(int t, struct bpf_verifier_env *env)
18276{
18277	static struct bpf_subprog_info *subprog;
18278	int subprog_start, subprog_end;
18279	struct bpf_iarray *jt;
18280	int i;
18281
18282	subprog = bpf_find_containing_subprog(env, t);
18283	subprog_start = subprog->start;
18284	subprog_end = (subprog + 1)->start;
18285	jt = jt_from_subprog(env, subprog_start, subprog_end);
18286	if (IS_ERR(jt))
18287		return jt;
18288
18289	/* Check that the every element of the jump table fits within the given subprogram */
18290	for (i = 0; i < jt->cnt; i++) {
18291		if (jt->items[i] < subprog_start || jt->items[i] >= subprog_end) {
18292			verbose(env, "jump table for insn %d points outside of the subprog [%u,%u]\n",
18293					t, subprog_start, subprog_end);
18294			kvfree(jt);
18295			return ERR_PTR(-EINVAL);
18296		}
18297	}
18298
18299	return jt;
18300}
18301
18302/* "conditional jump with N edges" */
18303static int visit_gotox_insn(int t, struct bpf_verifier_env *env)
18304{
18305	int *insn_stack = env->cfg.insn_stack;
18306	int *insn_state = env->cfg.insn_state;
18307	bool keep_exploring = false;
18308	struct bpf_iarray *jt;
18309	int i, w;
18310
18311	jt = env->insn_aux_data[t].jt;
18312	if (!jt) {
18313		jt = create_jt(t, env);
18314		if (IS_ERR(jt))
18315			return PTR_ERR(jt);
18316
18317		env->insn_aux_data[t].jt = jt;
18318	}
18319
18320	mark_prune_point(env, t);
18321	for (i = 0; i < jt->cnt; i++) {
18322		w = jt->items[i];
18323		if (w < 0 || w >= env->prog->len) {
18324			verbose(env, "indirect jump out of range from insn %d to %d\n", t, w);
18325			return -EINVAL;
18326		}
18327
18328		mark_jmp_point(env, w);
18329
18330		/* EXPLORED || DISCOVERED */
18331		if (insn_state[w])
18332			continue;
18333
18334		if (env->cfg.cur_stack >= env->prog->len)
18335			return -E2BIG;
18336
18337		insn_stack[env->cfg.cur_stack++] = w;
18338		insn_state[w] |= DISCOVERED;
18339		keep_exploring = true;
18340	}
18341
18342	return keep_exploring ? KEEP_EXPLORING : DONE_EXPLORING;
18343}
18344
18345static int visit_tailcall_insn(struct bpf_verifier_env *env, int t)
18346{
18347	static struct bpf_subprog_info *subprog;
18348	struct bpf_iarray *jt;
18349
18350	if (env->insn_aux_data[t].jt)
18351		return 0;
18352
18353	jt = iarray_realloc(NULL, 2);
18354	if (!jt)
18355		return -ENOMEM;
18356
18357	subprog = bpf_find_containing_subprog(env, t);
18358	jt->items[0] = t + 1;
18359	jt->items[1] = subprog->exit_idx;
18360	env->insn_aux_data[t].jt = jt;
18361	return 0;
18362}
18363
18364/* Visits the instruction at index t and returns one of the following:
18365 *  < 0 - an error occurred
18366 *  DONE_EXPLORING - the instruction was fully explored
18367 *  KEEP_EXPLORING - there is still work to be done before it is fully explored
18368 */
18369static int visit_insn(int t, struct bpf_verifier_env *env)
18370{
18371	struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t];
18372	int ret, off, insn_sz;
18373
18374	if (bpf_pseudo_func(insn))
18375		return visit_func_call_insn(t, insns, env, true);
18376
18377	/* All non-branch instructions have a single fall-through edge. */
18378	if (BPF_CLASS(insn->code) != BPF_JMP &&
18379	    BPF_CLASS(insn->code) != BPF_JMP32) {
18380		insn_sz = bpf_is_ldimm64(insn) ? 2 : 1;
18381		return push_insn(t, t + insn_sz, FALLTHROUGH, env);
18382	}
18383
18384	switch (BPF_OP(insn->code)) {
18385	case BPF_EXIT:
18386		return DONE_EXPLORING;
18387
18388	case BPF_CALL:
18389		if (is_async_callback_calling_insn(insn))
18390			/* Mark this call insn as a prune point to trigger
18391			 * is_state_visited() check before call itself is
18392			 * processed by __check_func_call(). Otherwise new
18393			 * async state will be pushed for further exploration.
18394			 */
18395			mark_prune_point(env, t);
18396		/* For functions that invoke callbacks it is not known how many times
18397		 * callback would be called. Verifier models callback calling functions
18398		 * by repeatedly visiting callback bodies and returning to origin call
18399		 * instruction.
18400		 * In order to stop such iteration verifier needs to identify when a
18401		 * state identical some state from a previous iteration is reached.
18402		 * Check below forces creation of checkpoint before callback calling
18403		 * instruction to allow search for such identical states.
18404		 */
18405		if (is_sync_callback_calling_insn(insn)) {
18406			mark_calls_callback(env, t);
18407			mark_force_checkpoint(env, t);
18408			mark_prune_point(env, t);
18409			mark_jmp_point(env, t);
18410		}
18411		if (bpf_helper_call(insn)) {
18412			const struct bpf_func_proto *fp;
18413
18414			ret = get_helper_proto(env, insn->imm, &fp);
18415			/* If called in a non-sleepable context program will be
18416			 * rejected anyway, so we should end up with precise
18417			 * sleepable marks on subprogs, except for dead code
18418			 * elimination.
18419			 */
18420			if (ret == 0 && fp->might_sleep)
18421				mark_subprog_might_sleep(env, t);
18422			if (bpf_helper_changes_pkt_data(insn->imm))
18423				mark_subprog_changes_pkt_data(env, t);
18424			if (insn->imm == BPF_FUNC_tail_call)
18425				visit_tailcall_insn(env, t);
18426		} else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
18427			struct bpf_kfunc_call_arg_meta meta;
18428
18429			ret = fetch_kfunc_meta(env, insn, &meta, NULL);
18430			if (ret == 0 && is_iter_next_kfunc(&meta)) {
18431				mark_prune_point(env, t);
18432				/* Checking and saving state checkpoints at iter_next() call
18433				 * is crucial for fast convergence of open-coded iterator loop
18434				 * logic, so we need to force it. If we don't do that,
18435				 * is_state_visited() might skip saving a checkpoint, causing
18436				 * unnecessarily long sequence of not checkpointed
18437				 * instructions and jumps, leading to exhaustion of jump
18438				 * history buffer, and potentially other undesired outcomes.
18439				 * It is expected that with correct open-coded iterators
18440				 * convergence will happen quickly, so we don't run a risk of
18441				 * exhausting memory.
18442				 */
18443				mark_force_checkpoint(env, t);
18444			}
18445			/* Same as helpers, if called in a non-sleepable context
18446			 * program will be rejected anyway, so we should end up
18447			 * with precise sleepable marks on subprogs, except for
18448			 * dead code elimination.
18449			 */
18450			if (ret == 0 && is_kfunc_sleepable(&meta))
18451				mark_subprog_might_sleep(env, t);
18452			if (ret == 0 && is_kfunc_pkt_changing(&meta))
18453				mark_subprog_changes_pkt_data(env, t);
18454		}
18455		return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL);
18456
18457	case BPF_JA:
18458		if (BPF_SRC(insn->code) == BPF_X)
18459			return visit_gotox_insn(t, env);
18460
18461		if (BPF_CLASS(insn->code) == BPF_JMP)
18462			off = insn->off;
18463		else
18464			off = insn->imm;
18465
18466		/* unconditional jump with single edge */
18467		ret = push_insn(t, t + off + 1, FALLTHROUGH, env);
18468		if (ret)
18469			return ret;
18470
18471		mark_prune_point(env, t + off + 1);
18472		mark_jmp_point(env, t + off + 1);
18473
18474		return ret;
18475
18476	default:
18477		/* conditional jump with two edges */
18478		mark_prune_point(env, t);
18479		if (is_may_goto_insn(insn))
18480			mark_force_checkpoint(env, t);
18481
18482		ret = push_insn(t, t + 1, FALLTHROUGH, env);
18483		if (ret)
18484			return ret;
18485
18486		return push_insn(t, t + insn->off + 1, BRANCH, env);
18487	}
18488}
18489
18490/* non-recursive depth-first-search to detect loops in BPF program
18491 * loop == back-edge in directed graph
18492 */
18493static int check_cfg(struct bpf_verifier_env *env)
18494{
18495	int insn_cnt = env->prog->len;
18496	int *insn_stack, *insn_state;
18497	int ex_insn_beg, i, ret = 0;
18498
18499	insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
18500	if (!insn_state)
18501		return -ENOMEM;
18502
18503	insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
18504	if (!insn_stack) {
18505		kvfree(insn_state);
18506		return -ENOMEM;
18507	}
18508
18509	ex_insn_beg = env->exception_callback_subprog
18510		      ? env->subprog_info[env->exception_callback_subprog].start
18511		      : 0;
18512
18513	insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */
18514	insn_stack[0] = 0; /* 0 is the first instruction */
18515	env->cfg.cur_stack = 1;
18516
18517walk_cfg:
18518	while (env->cfg.cur_stack > 0) {
18519		int t = insn_stack[env->cfg.cur_stack - 1];
18520
18521		ret = visit_insn(t, env);
18522		switch (ret) {
18523		case DONE_EXPLORING:
18524			insn_state[t] = EXPLORED;
18525			env->cfg.cur_stack--;
18526			break;
18527		case KEEP_EXPLORING:
18528			break;
18529		default:
18530			if (ret > 0) {
18531				verifier_bug(env, "visit_insn internal bug");
18532				ret = -EFAULT;
18533			}
18534			goto err_free;
18535		}
18536	}
18537
18538	if (env->cfg.cur_stack < 0) {
18539		verifier_bug(env, "pop stack internal bug");
18540		ret = -EFAULT;
18541		goto err_free;
18542	}
18543
18544	if (ex_insn_beg && insn_state[ex_insn_beg] != EXPLORED) {
18545		insn_state[ex_insn_beg] = DISCOVERED;
18546		insn_stack[0] = ex_insn_beg;
18547		env->cfg.cur_stack = 1;
18548		goto walk_cfg;
18549	}
18550
18551	for (i = 0; i < insn_cnt; i++) {
18552		struct bpf_insn *insn = &env->prog->insnsi[i];
18553
18554		if (insn_state[i] != EXPLORED) {
18555			verbose(env, "unreachable insn %d\n", i);
18556			ret = -EINVAL;
18557			goto err_free;
18558		}
18559		if (bpf_is_ldimm64(insn)) {
18560			if (insn_state[i + 1] != 0) {
18561				verbose(env, "jump into the middle of ldimm64 insn %d\n", i);
18562				ret = -EINVAL;
18563				goto err_free;
18564			}
18565			i++; /* skip second half of ldimm64 */
18566		}
18567	}
18568	ret = 0; /* cfg looks good */
18569	env->prog->aux->changes_pkt_data = env->subprog_info[0].changes_pkt_data;
18570	env->prog->aux->might_sleep = env->subprog_info[0].might_sleep;
18571
18572err_free:
18573	kvfree(insn_state);
18574	kvfree(insn_stack);
18575	env->cfg.insn_state = env->cfg.insn_stack = NULL;
18576	return ret;
18577}
18578
18579/*
18580 * For each subprogram 'i' fill array env->cfg.insn_subprogram sub-range
18581 * [env->subprog_info[i].postorder_start, env->subprog_info[i+1].postorder_start)
18582 * with indices of 'i' instructions in postorder.
18583 */
18584static int compute_postorder(struct bpf_verifier_env *env)
18585{
18586	u32 cur_postorder, i, top, stack_sz, s;
18587	int *stack = NULL, *postorder = NULL, *state = NULL;
18588	struct bpf_iarray *succ;
18589
18590	postorder = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT);
18591	state = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT);
18592	stack = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT);
18593	if (!postorder || !state || !stack) {
18594		kvfree(postorder);
18595		kvfree(state);
18596		kvfree(stack);
18597		return -ENOMEM;
18598	}
18599	cur_postorder = 0;
18600	for (i = 0; i < env->subprog_cnt; i++) {
18601		env->subprog_info[i].postorder_start = cur_postorder;
18602		stack[0] = env->subprog_info[i].start;
18603		stack_sz = 1;
18604		do {
18605			top = stack[stack_sz - 1];
18606			state[top] |= DISCOVERED;
18607			if (state[top] & EXPLORED) {
18608				postorder[cur_postorder++] = top;
18609				stack_sz--;
18610				continue;
18611			}
18612			succ = bpf_insn_successors(env, top);
18613			for (s = 0; s < succ->cnt; ++s) {
18614				if (!state[succ->items[s]]) {
18615					stack[stack_sz++] = succ->items[s];
18616					state[succ->items[s]] |= DISCOVERED;
18617				}
18618			}
18619			state[top] |= EXPLORED;
18620		} while (stack_sz);
18621	}
18622	env->subprog_info[i].postorder_start = cur_postorder;
18623	env->cfg.insn_postorder = postorder;
18624	env->cfg.cur_postorder = cur_postorder;
18625	kvfree(stack);
18626	kvfree(state);
18627	return 0;
18628}
18629
18630static int check_abnormal_return(struct bpf_verifier_env *env)
18631{
18632	int i;
18633
18634	for (i = 1; i < env->subprog_cnt; i++) {
18635		if (env->subprog_info[i].has_ld_abs) {
18636			verbose(env, "LD_ABS is not allowed in subprogs without BTF\n");
18637			return -EINVAL;
18638		}
18639		if (env->subprog_info[i].has_tail_call) {
18640			verbose(env, "tail_call is not allowed in subprogs without BTF\n");
18641			return -EINVAL;
18642		}
18643	}
18644	return 0;
18645}
18646
18647/* The minimum supported BTF func info size */
18648#define MIN_BPF_FUNCINFO_SIZE	8
18649#define MAX_FUNCINFO_REC_SIZE	252
18650
18651static int check_btf_func_early(struct bpf_verifier_env *env,
18652				const union bpf_attr *attr,
18653				bpfptr_t uattr)
18654{
18655	u32 krec_size = sizeof(struct bpf_func_info);
18656	const struct btf_type *type, *func_proto;
18657	u32 i, nfuncs, urec_size, min_size;
18658	struct bpf_func_info *krecord;
18659	struct bpf_prog *prog;
18660	const struct btf *btf;
18661	u32 prev_offset = 0;
18662	bpfptr_t urecord;
18663	int ret = -ENOMEM;
18664
18665	nfuncs = attr->func_info_cnt;
18666	if (!nfuncs) {
18667		if (check_abnormal_return(env))
18668			return -EINVAL;
18669		return 0;
18670	}
18671
18672	urec_size = attr->func_info_rec_size;
18673	if (urec_size < MIN_BPF_FUNCINFO_SIZE ||
18674	    urec_size > MAX_FUNCINFO_REC_SIZE ||
18675	    urec_size % sizeof(u32)) {
18676		verbose(env, "invalid func info rec size %u\n", urec_size);
18677		return -EINVAL;
18678	}
18679
18680	prog = env->prog;
18681	btf = prog->aux->btf;
18682
18683	urecord = make_bpfptr(attr->func_info, uattr.is_kernel);
18684	min_size = min_t(u32, krec_size, urec_size);
18685
18686	krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
18687	if (!krecord)
18688		return -ENOMEM;
18689
18690	for (i = 0; i < nfuncs; i++) {
18691		ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
18692		if (ret) {
18693			if (ret == -E2BIG) {
18694				verbose(env, "nonzero tailing record in func info");
18695				/* set the size kernel expects so loader can zero
18696				 * out the rest of the record.
18697				 */
18698				if (copy_to_bpfptr_offset(uattr,
18699							  offsetof(union bpf_attr, func_info_rec_size),
18700							  &min_size, sizeof(min_size)))
18701					ret = -EFAULT;
18702			}
18703			goto err_free;
18704		}
18705
18706		if (copy_from_bpfptr(&krecord[i], urecord, min_size)) {
18707			ret = -EFAULT;
18708			goto err_free;
18709		}
18710
18711		/* check insn_off */
18712		ret = -EINVAL;
18713		if (i == 0) {
18714			if (krecord[i].insn_off) {
18715				verbose(env,
18716					"nonzero insn_off %u for the first func info record",
18717					krecord[i].insn_off);
18718				goto err_free;
18719			}
18720		} else if (krecord[i].insn_off <= prev_offset) {
18721			verbose(env,
18722				"same or smaller insn offset (%u) than previous func info record (%u)",
18723				krecord[i].insn_off, prev_offset);
18724			goto err_free;
18725		}
18726
18727		/* check type_id */
18728		type = btf_type_by_id(btf, krecord[i].type_id);
18729		if (!type || !btf_type_is_func(type)) {
18730			verbose(env, "invalid type id %d in func info",
18731				krecord[i].type_id);
18732			goto err_free;
18733		}
18734
18735		func_proto = btf_type_by_id(btf, type->type);
18736		if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto)))
18737			/* btf_func_check() already verified it during BTF load */
18738			goto err_free;
18739
18740		prev_offset = krecord[i].insn_off;
18741		bpfptr_add(&urecord, urec_size);
18742	}
18743
18744	prog->aux->func_info = krecord;
18745	prog->aux->func_info_cnt = nfuncs;
18746	return 0;
18747
18748err_free:
18749	kvfree(krecord);
18750	return ret;
18751}
18752
18753static int check_btf_func(struct bpf_verifier_env *env,
18754			  const union bpf_attr *attr,
18755			  bpfptr_t uattr)
18756{
18757	const struct btf_type *type, *func_proto, *ret_type;
18758	u32 i, nfuncs, urec_size;
18759	struct bpf_func_info *krecord;
18760	struct bpf_func_info_aux *info_aux = NULL;
18761	struct bpf_prog *prog;
18762	const struct btf *btf;
18763	bpfptr_t urecord;
18764	bool scalar_return;
18765	int ret = -ENOMEM;
18766
18767	nfuncs = attr->func_info_cnt;
18768	if (!nfuncs) {
18769		if (check_abnormal_return(env))
18770			return -EINVAL;
18771		return 0;
18772	}
18773	if (nfuncs != env->subprog_cnt) {
18774		verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");
18775		return -EINVAL;
18776	}
18777
18778	urec_size = attr->func_info_rec_size;
18779
18780	prog = env->prog;
18781	btf = prog->aux->btf;
18782
18783	urecord = make_bpfptr(attr->func_info, uattr.is_kernel);
18784
18785	krecord = prog->aux->func_info;
18786	info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
18787	if (!info_aux)
18788		return -ENOMEM;
18789
18790	for (i = 0; i < nfuncs; i++) {
18791		/* check insn_off */
18792		ret = -EINVAL;
18793
18794		if (env->subprog_info[i].start != krecord[i].insn_off) {
18795			verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
18796			goto err_free;
18797		}
18798
18799		/* Already checked type_id */
18800		type = btf_type_by_id(btf, krecord[i].type_id);
18801		info_aux[i].linkage = BTF_INFO_VLEN(type->info);
18802		/* Already checked func_proto */
18803		func_proto = btf_type_by_id(btf, type->type);
18804
18805		ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL);
18806		scalar_return =
18807			btf_type_is_small_int(ret_type) || btf_is_any_enum(ret_type);
18808		if (i && !scalar_return && env->subprog_info[i].has_ld_abs) {
18809			verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n");
18810			goto err_free;
18811		}
18812		if (i && !scalar_return && env->subprog_info[i].has_tail_call) {
18813			verbose(env, "tail_call is only allowed in functions that return 'int'.\n");
18814			goto err_free;
18815		}
18816
18817		bpfptr_add(&urecord, urec_size);
18818	}
18819
18820	prog->aux->func_info_aux = info_aux;
18821	return 0;
18822
18823err_free:
18824	kfree(info_aux);
18825	return ret;
18826}
18827
18828static void adjust_btf_func(struct bpf_verifier_env *env)
18829{
18830	struct bpf_prog_aux *aux = env->prog->aux;
18831	int i;
18832
18833	if (!aux->func_info)
18834		return;
18835
18836	/* func_info is not available for hidden subprogs */
18837	for (i = 0; i < env->subprog_cnt - env->hidden_subprog_cnt; i++)
18838		aux->func_info[i].insn_off = env->subprog_info[i].start;
18839}
18840
18841#define MIN_BPF_LINEINFO_SIZE	offsetofend(struct bpf_line_info, line_col)
18842#define MAX_LINEINFO_REC_SIZE	MAX_FUNCINFO_REC_SIZE
18843
18844static int check_btf_line(struct bpf_verifier_env *env,
18845			  const union bpf_attr *attr,
18846			  bpfptr_t uattr)
18847{
18848	u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0;
18849	struct bpf_subprog_info *sub;
18850	struct bpf_line_info *linfo;
18851	struct bpf_prog *prog;
18852	const struct btf *btf;
18853	bpfptr_t ulinfo;
18854	int err;
18855
18856	nr_linfo = attr->line_info_cnt;
18857	if (!nr_linfo)
18858		return 0;
18859	if (nr_linfo > INT_MAX / sizeof(struct bpf_line_info))
18860		return -EINVAL;
18861
18862	rec_size = attr->line_info_rec_size;
18863	if (rec_size < MIN_BPF_LINEINFO_SIZE ||
18864	    rec_size > MAX_LINEINFO_REC_SIZE ||
18865	    rec_size & (sizeof(u32) - 1))
18866		return -EINVAL;
18867
18868	/* Need to zero it in case the userspace may
18869	 * pass in a smaller bpf_line_info object.
18870	 */
18871	linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info),
18872			 GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
18873	if (!linfo)
18874		return -ENOMEM;
18875
18876	prog = env->prog;
18877	btf = prog->aux->btf;
18878
18879	s = 0;
18880	sub = env->subprog_info;
18881	ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel);
18882	expected_size = sizeof(struct bpf_line_info);
18883	ncopy = min_t(u32, expected_size, rec_size);
18884	for (i = 0; i < nr_linfo; i++) {
18885		err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size);
18886		if (err) {
18887			if (err == -E2BIG) {
18888				verbose(env, "nonzero tailing record in line_info");
18889				if (copy_to_bpfptr_offset(uattr,
18890							  offsetof(union bpf_attr, line_info_rec_size),
18891							  &expected_size, sizeof(expected_size)))
18892					err = -EFAULT;
18893			}
18894			goto err_free;
18895		}
18896
18897		if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) {
18898			err = -EFAULT;
18899			goto err_free;
18900		}
18901
18902		/*
18903		 * Check insn_off to ensure
18904		 * 1) strictly increasing AND
18905		 * 2) bounded by prog->len
18906		 *
18907		 * The linfo[0].insn_off == 0 check logically falls into
18908		 * the later "missing bpf_line_info for func..." case
18909		 * because the first linfo[0].insn_off must be the
18910		 * first sub also and the first sub must have
18911		 * subprog_info[0].start == 0.
18912		 */
18913		if ((i && linfo[i].insn_off <= prev_offset) ||
18914		    linfo[i].insn_off >= prog->len) {
18915			verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",
18916				i, linfo[i].insn_off, prev_offset,
18917				prog->len);
18918			err = -EINVAL;
18919			goto err_free;
18920		}
18921
18922		if (!prog->insnsi[linfo[i].insn_off].code) {
18923			verbose(env,
18924				"Invalid insn code at line_info[%u].insn_off\n",
18925				i);
18926			err = -EINVAL;
18927			goto err_free;
18928		}
18929
18930		if (!btf_name_by_offset(btf, linfo[i].line_off) ||
18931		    !btf_name_by_offset(btf, linfo[i].file_name_off)) {
18932			verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);
18933			err = -EINVAL;
18934			goto err_free;
18935		}
18936
18937		if (s != env->subprog_cnt) {
18938			if (linfo[i].insn_off == sub[s].start) {
18939				sub[s].linfo_idx = i;
18940				s++;
18941			} else if (sub[s].start < linfo[i].insn_off) {
18942				verbose(env, "missing bpf_line_info for func#%u\n", s);
18943				err = -EINVAL;
18944				goto err_free;
18945			}
18946		}
18947
18948		prev_offset = linfo[i].insn_off;
18949		bpfptr_add(&ulinfo, rec_size);
18950	}
18951
18952	if (s != env->subprog_cnt) {
18953		verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",
18954			env->subprog_cnt - s, s);
18955		err = -EINVAL;
18956		goto err_free;
18957	}
18958
18959	prog->aux->linfo = linfo;
18960	prog->aux->nr_linfo = nr_linfo;
18961
18962	return 0;
18963
18964err_free:
18965	kvfree(linfo);
18966	return err;
18967}
18968
18969#define MIN_CORE_RELO_SIZE	sizeof(struct bpf_core_relo)
18970#define MAX_CORE_RELO_SIZE	MAX_FUNCINFO_REC_SIZE
18971
18972static int check_core_relo(struct bpf_verifier_env *env,
18973			   const union bpf_attr *attr,
18974			   bpfptr_t uattr)
18975{
18976	u32 i, nr_core_relo, ncopy, expected_size, rec_size;
18977	struct bpf_core_relo core_relo = {};
18978	struct bpf_prog *prog = env->prog;
18979	const struct btf *btf = prog->aux->btf;
18980	struct bpf_core_ctx ctx = {
18981		.log = &env->log,
18982		.btf = btf,
18983	};
18984	bpfptr_t u_core_relo;
18985	int err;
18986
18987	nr_core_relo = attr->core_relo_cnt;
18988	if (!nr_core_relo)
18989		return 0;
18990	if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo))
18991		return -EINVAL;
18992
18993	rec_size = attr->core_relo_rec_size;
18994	if (rec_size < MIN_CORE_RELO_SIZE ||
18995	    rec_size > MAX_CORE_RELO_SIZE ||
18996	    rec_size % sizeof(u32))
18997		return -EINVAL;
18998
18999	u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel);
19000	expected_size = sizeof(struct bpf_core_relo);
19001	ncopy = min_t(u32, expected_size, rec_size);
19002
19003	/* Unlike func_info and line_info, copy and apply each CO-RE
19004	 * relocation record one at a time.
19005	 */
19006	for (i = 0; i < nr_core_relo; i++) {
19007		/* future proofing when sizeof(bpf_core_relo) changes */
19008		err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size);
19009		if (err) {
19010			if (err == -E2BIG) {
19011				verbose(env, "nonzero tailing record in core_relo");
19012				if (copy_to_bpfptr_offset(uattr,
19013							  offsetof(union bpf_attr, core_relo_rec_size),
19014							  &expected_size, sizeof(expected_size)))
19015					err = -EFAULT;
19016			}
19017			break;
19018		}
19019
19020		if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) {
19021			err = -EFAULT;
19022			break;
19023		}
19024
19025		if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) {
19026			verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n",
19027				i, core_relo.insn_off, prog->len);
19028			err = -EINVAL;
19029			break;
19030		}
19031
19032		err = bpf_core_apply(&ctx, &core_relo, i,
19033				     &prog->insnsi[core_relo.insn_off / 8]);
19034		if (err)
19035			break;
19036		bpfptr_add(&u_core_relo, rec_size);
19037	}
19038	return err;
19039}
19040
19041static int check_btf_info_early(struct bpf_verifier_env *env,
19042				const union bpf_attr *attr,
19043				bpfptr_t uattr)
19044{
19045	struct btf *btf;
19046	int err;
19047
19048	if (!attr->func_info_cnt && !attr->line_info_cnt) {
19049		if (check_abnormal_return(env))
19050			return -EINVAL;
19051		return 0;
19052	}
19053
19054	btf = btf_get_by_fd(attr->prog_btf_fd);
19055	if (IS_ERR(btf))
19056		return PTR_ERR(btf);
19057	if (btf_is_kernel(btf)) {
19058		btf_put(btf);
19059		return -EACCES;
19060	}
19061	env->prog->aux->btf = btf;
19062
19063	err = check_btf_func_early(env, attr, uattr);
19064	if (err)
19065		return err;
19066	return 0;
19067}
19068
19069static int check_btf_info(struct bpf_verifier_env *env,
19070			  const union bpf_attr *attr,
19071			  bpfptr_t uattr)
19072{
19073	int err;
19074
19075	if (!attr->func_info_cnt && !attr->line_info_cnt) {
19076		if (check_abnormal_return(env))
19077			return -EINVAL;
19078		return 0;
19079	}
19080
19081	err = check_btf_func(env, attr, uattr);
19082	if (err)
19083		return err;
19084
19085	err = check_btf_line(env, attr, uattr);
19086	if (err)
19087		return err;
19088
19089	err = check_core_relo(env, attr, uattr);
19090	if (err)
19091		return err;
19092
19093	return 0;
19094}
19095
19096/* check %cur's range satisfies %old's */
19097static bool range_within(const struct bpf_reg_state *old,
19098			 const struct bpf_reg_state *cur)
19099{
19100	return old->umin_value <= cur->umin_value &&
19101	       old->umax_value >= cur->umax_value &&
19102	       old->smin_value <= cur->smin_value &&
19103	       old->smax_value >= cur->smax_value &&
19104	       old->u32_min_value <= cur->u32_min_value &&
19105	       old->u32_max_value >= cur->u32_max_value &&
19106	       old->s32_min_value <= cur->s32_min_value &&
19107	       old->s32_max_value >= cur->s32_max_value;
19108}
19109
19110/* If in the old state two registers had the same id, then they need to have
19111 * the same id in the new state as well.  But that id could be different from
19112 * the old state, so we need to track the mapping from old to new ids.
19113 * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent
19114 * regs with old id 5 must also have new id 9 for the new state to be safe.  But
19115 * regs with a different old id could still have new id 9, we don't care about
19116 * that.
19117 * So we look through our idmap to see if this old id has been seen before.  If
19118 * so, we require the new id to match; otherwise, we add the id pair to the map.
19119 */
19120static bool check_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap)
19121{
19122	struct bpf_id_pair *map = idmap->map;
19123	unsigned int i;
19124
19125	/* either both IDs should be set or both should be zero */
19126	if (!!old_id != !!cur_id)
19127		return false;
19128
19129	if (old_id == 0) /* cur_id == 0 as well */
19130		return true;
19131
19132	for (i = 0; i < BPF_ID_MAP_SIZE; i++) {
19133		if (!map[i].old) {
19134			/* Reached an empty slot; haven't seen this id before */
19135			map[i].old = old_id;
19136			map[i].cur = cur_id;
19137			return true;
19138		}
19139		if (map[i].old == old_id)
19140			return map[i].cur == cur_id;
19141		if (map[i].cur == cur_id)
19142			return false;
19143	}
19144	/* We ran out of idmap slots, which should be impossible */
19145	WARN_ON_ONCE(1);
19146	return false;
19147}
19148
19149/* Similar to check_ids(), but allocate a unique temporary ID
19150 * for 'old_id' or 'cur_id' of zero.
19151 * This makes pairs like '0 vs unique ID', 'unique ID vs 0' valid.
19152 */
19153static bool check_scalar_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap)
19154{
19155	old_id = old_id ? old_id : ++idmap->tmp_id_gen;
19156	cur_id = cur_id ? cur_id : ++idmap->tmp_id_gen;
19157
19158	return check_ids(old_id, cur_id, idmap);
19159}
19160
19161static void clean_func_state(struct bpf_verifier_env *env,
19162			     struct bpf_func_state *st,
19163			     u32 ip)
19164{
19165	u16 live_regs = env->insn_aux_data[ip].live_regs_before;
19166	int i, j;
19167
19168	for (i = 0; i < BPF_REG_FP; i++) {
19169		/* liveness must not touch this register anymore */
19170		if (!(live_regs & BIT(i)))
19171			/* since the register is unused, clear its state
19172			 * to make further comparison simpler
19173			 */
19174			__mark_reg_not_init(env, &st->regs[i]);
19175	}
19176
19177	for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) {
19178		if (!bpf_stack_slot_alive(env, st->frameno, i)) {
19179			__mark_reg_not_init(env, &st->stack[i].spilled_ptr);
19180			for (j = 0; j < BPF_REG_SIZE; j++)
19181				st->stack[i].slot_type[j] = STACK_INVALID;
19182		}
19183	}
19184}
19185
19186static void clean_verifier_state(struct bpf_verifier_env *env,
19187				 struct bpf_verifier_state *st)
19188{
19189	int i, ip;
19190
19191	bpf_live_stack_query_init(env, st);
19192	st->cleaned = true;
19193	for (i = 0; i <= st->curframe; i++) {
19194		ip = frame_insn_idx(st, i);
19195		clean_func_state(env, st->frame[i], ip);
19196	}
19197}
19198
19199/* the parentage chains form a tree.
19200 * the verifier states are added to state lists at given insn and
19201 * pushed into state stack for future exploration.
19202 * when the verifier reaches bpf_exit insn some of the verifier states
19203 * stored in the state lists have their final liveness state already,
19204 * but a lot of states will get revised from liveness point of view when
19205 * the verifier explores other branches.
19206 * Example:
19207 * 1: *(u64)(r10 - 8) = 1
19208 * 2: if r1 == 100 goto pc+1
19209 * 3: *(u64)(r10 - 8) = 2
19210 * 4: r0 = *(u64)(r10 - 8)
19211 * 5: exit
19212 * when the verifier reaches exit insn the stack slot -8 in the state list of
19213 * insn 2 is not yet marked alive. Then the verifier pops the other_branch
19214 * of insn 2 and goes exploring further. After the insn 4 read, liveness
19215 * analysis would propagate read mark for -8 at insn 2.
19216 *
19217 * Since the verifier pushes the branch states as it sees them while exploring
19218 * the program the condition of walking the branch instruction for the second
19219 * time means that all states below this branch were already explored and
19220 * their final liveness marks are already propagated.
19221 * Hence when the verifier completes the search of state list in is_state_visited()
19222 * we can call this clean_live_states() function to clear dead the registers and stack
19223 * slots to simplify state merging.
19224 *
19225 * Important note here that walking the same branch instruction in the callee
19226 * doesn't meant that the states are DONE. The verifier has to compare
19227 * the callsites
19228 */
19229static void clean_live_states(struct bpf_verifier_env *env, int insn,
19230			      struct bpf_verifier_state *cur)
19231{
19232	struct bpf_verifier_state_list *sl;
19233	struct list_head *pos, *head;
19234
19235	head = explored_state(env, insn);
19236	list_for_each(pos, head) {
19237		sl = container_of(pos, struct bpf_verifier_state_list, node);
19238		if (sl->state.branches)
19239			continue;
19240		if (sl->state.insn_idx != insn ||
19241		    !same_callsites(&sl->state, cur))
19242			continue;
19243		if (sl->state.cleaned)
19244			/* all regs in this state in all frames were already marked */
19245			continue;
19246		if (incomplete_read_marks(env, &sl->state))
19247			continue;
19248		clean_verifier_state(env, &sl->state);
19249	}
19250}
19251
19252static bool regs_exact(const struct bpf_reg_state *rold,
19253		       const struct bpf_reg_state *rcur,
19254		       struct bpf_idmap *idmap)
19255{
19256	return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
19257	       check_ids(rold->id, rcur->id, idmap) &&
19258	       check_ids(rold->ref_obj_id, rcur->ref_obj_id, idmap);
19259}
19260
19261enum exact_level {
19262	NOT_EXACT,
19263	EXACT,
19264	RANGE_WITHIN
19265};
19266
19267/* Returns true if (rold safe implies rcur safe) */
19268static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
19269		    struct bpf_reg_state *rcur, struct bpf_idmap *idmap,
19270		    enum exact_level exact)
19271{
19272	if (exact == EXACT)
19273		return regs_exact(rold, rcur, idmap);
19274
19275	if (rold->type == NOT_INIT) {
19276		if (exact == NOT_EXACT || rcur->type == NOT_INIT)
19277			/* explored state can't have used this */
19278			return true;
19279	}
19280
19281	/* Enforce that register types have to match exactly, including their
19282	 * modifiers (like PTR_MAYBE_NULL, MEM_RDONLY, etc), as a general
19283	 * rule.
19284	 *
19285	 * One can make a point that using a pointer register as unbounded
19286	 * SCALAR would be technically acceptable, but this could lead to
19287	 * pointer leaks because scalars are allowed to leak while pointers
19288	 * are not. We could make this safe in special cases if root is
19289	 * calling us, but it's probably not worth the hassle.
19290	 *
19291	 * Also, register types that are *not* MAYBE_NULL could technically be
19292	 * safe to use as their MAYBE_NULL variants (e.g., PTR_TO_MAP_VALUE
19293	 * is safe to be used as PTR_TO_MAP_VALUE_OR_NULL, provided both point
19294	 * to the same map).
19295	 * However, if the old MAYBE_NULL register then got NULL checked,
19296	 * doing so could have affected others with the same id, and we can't
19297	 * check for that because we lost the id when we converted to
19298	 * a non-MAYBE_NULL variant.
19299	 * So, as a general rule we don't allow mixing MAYBE_NULL and
19300	 * non-MAYBE_NULL registers as well.
19301	 */
19302	if (rold->type != rcur->type)
19303		return false;
19304
19305	switch (base_type(rold->type)) {
19306	case SCALAR_VALUE:
19307		if (env->explore_alu_limits) {
19308			/* explore_alu_limits disables tnum_in() and range_within()
19309			 * logic and requires everything to be strict
19310			 */
19311			return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
19312			       check_scalar_ids(rold->id, rcur->id, idmap);
19313		}
19314		if (!rold->precise && exact == NOT_EXACT)
19315			return true;
19316		if ((rold->id & BPF_ADD_CONST) != (rcur->id & BPF_ADD_CONST))
19317			return false;
19318		if ((rold->id & BPF_ADD_CONST) && (rold->off != rcur->off))
19319			return false;
19320		/* Why check_ids() for scalar registers?
19321		 *
19322		 * Consider the following BPF code:
19323		 *   1: r6 = ... unbound scalar, ID=a ...
19324		 *   2: r7 = ... unbound scalar, ID=b ...
19325		 *   3: if (r6 > r7) goto +1
19326		 *   4: r6 = r7
19327		 *   5: if (r6 > X) goto ...
19328		 *   6: ... memory operation using r7 ...
19329		 *
19330		 * First verification path is [1-6]:
19331		 * - at (4) same bpf_reg_state::id (b) would be assigned to r6 and r7;
19332		 * - at (5) r6 would be marked <= X, sync_linked_regs() would also mark
19333		 *   r7 <= X, because r6 and r7 share same id.
19334		 * Next verification path is [1-4, 6].
19335		 *
19336		 * Instruction (6) would be reached in two states:
19337		 *   I.  r6{.id=b}, r7{.id=b} via path 1-6;
19338		 *   II. r6{.id=a}, r7{.id=b} via path 1-4, 6.
19339		 *
19340		 * Use check_ids() to distinguish these states.
19341		 * ---
19342		 * Also verify that new value satisfies old value range knowledge.
19343		 */
19344		return range_within(rold, rcur) &&
19345		       tnum_in(rold->var_off, rcur->var_off) &&
19346		       check_scalar_ids(rold->id, rcur->id, idmap);
19347	case PTR_TO_MAP_KEY:
19348	case PTR_TO_MAP_VALUE:
19349	case PTR_TO_MEM:
19350	case PTR_TO_BUF:
19351	case PTR_TO_TP_BUFFER:
19352		/* If the new min/max/var_off satisfy the old ones and
19353		 * everything else matches, we are OK.
19354		 */
19355		return memcmp(rold, rcur, offsetof(struct bpf_reg_state, var_off)) == 0 &&
19356		       range_within(rold, rcur) &&
19357		       tnum_in(rold->var_off, rcur->var_off) &&
19358		       check_ids(rold->id, rcur->id, idmap) &&
19359		       check_ids(rold->ref_obj_id, rcur->ref_obj_id, idmap);
19360	case PTR_TO_PACKET_META:
19361	case PTR_TO_PACKET:
19362		/* We must have at least as much range as the old ptr
19363		 * did, so that any accesses which were safe before are
19364		 * still safe.  This is true even if old range < old off,
19365		 * since someone could have accessed through (ptr - k), or
19366		 * even done ptr -= k in a register, to get a safe access.
19367		 */
19368		if (rold->range < 0 || rcur->range < 0) {
19369			/* special case for [BEYOND|AT]_PKT_END */
19370			if (rold->range != rcur->range)
19371				return false;
19372		} else if (rold->range > rcur->range) {
19373			return false;
19374		}
19375		/* If the offsets don't match, we can't trust our alignment;
19376		 * nor can we be sure that we won't fall out of range.
19377		 */
19378		if (rold->off != rcur->off)
19379			return false;
19380		/* id relations must be preserved */
19381		if (!check_ids(rold->id, rcur->id, idmap))
19382			return false;
19383		/* new val must satisfy old val knowledge */
19384		return range_within(rold, rcur) &&
19385		       tnum_in(rold->var_off, rcur->var_off);
19386	case PTR_TO_STACK:
19387		/* two stack pointers are equal only if they're pointing to
19388		 * the same stack frame, since fp-8 in foo != fp-8 in bar
19389		 */
19390		return regs_exact(rold, rcur, idmap) && rold->frameno == rcur->frameno;
19391	case PTR_TO_ARENA:
19392		return true;
19393	case PTR_TO_INSN:
19394		return memcmp(rold, rcur, offsetof(struct bpf_reg_state, var_off)) == 0 &&
19395			rold->off == rcur->off && range_within(rold, rcur) &&
19396			tnum_in(rold->var_off, rcur->var_off);
19397	default:
19398		return regs_exact(rold, rcur, idmap);
19399	}
19400}
19401
19402static struct bpf_reg_state unbound_reg;
19403
19404static __init int unbound_reg_init(void)
19405{
19406	__mark_reg_unknown_imprecise(&unbound_reg);
19407	return 0;
19408}
19409late_initcall(unbound_reg_init);
19410
19411static bool is_stack_all_misc(struct bpf_verifier_env *env,
19412			      struct bpf_stack_state *stack)
19413{
19414	u32 i;
19415
19416	for (i = 0; i < ARRAY_SIZE(stack->slot_type); ++i) {
19417		if ((stack->slot_type[i] == STACK_MISC) ||
19418		    (stack->slot_type[i] == STACK_INVALID && env->allow_uninit_stack))
19419			continue;
19420		return false;
19421	}
19422
19423	return true;
19424}
19425
19426static struct bpf_reg_state *scalar_reg_for_stack(struct bpf_verifier_env *env,
19427						  struct bpf_stack_state *stack)
19428{
19429	if (is_spilled_scalar_reg64(stack))
19430		return &stack->spilled_ptr;
19431
19432	if (is_stack_all_misc(env, stack))
19433		return &unbound_reg;
19434
19435	return NULL;
19436}
19437
19438static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
19439		      struct bpf_func_state *cur, struct bpf_idmap *idmap,
19440		      enum exact_level exact)
19441{
19442	int i, spi;
19443
19444	/* walk slots of the explored stack and ignore any additional
19445	 * slots in the current stack, since explored(safe) state
19446	 * didn't use them
19447	 */
19448	for (i = 0; i < old->allocated_stack; i++) {
19449		struct bpf_reg_state *old_reg, *cur_reg;
19450
19451		spi = i / BPF_REG_SIZE;
19452
19453		if (exact != NOT_EXACT &&
19454		    (i >= cur->allocated_stack ||
19455		     old->stack[spi].slot_type[i % BPF_REG_SIZE] !=
19456		     cur->stack[spi].slot_type[i % BPF_REG_SIZE]))
19457			return false;
19458
19459		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID)
19460			continue;
19461
19462		if (env->allow_uninit_stack &&
19463		    old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC)
19464			continue;
19465
19466		/* explored stack has more populated slots than current stack
19467		 * and these slots were used
19468		 */
19469		if (i >= cur->allocated_stack)
19470			return false;
19471
19472		/* 64-bit scalar spill vs all slots MISC and vice versa.
19473		 * Load from all slots MISC produces unbound scalar.
19474		 * Construct a fake register for such stack and call
19475		 * regsafe() to ensure scalar ids are compared.
19476		 */
19477		old_reg = scalar_reg_for_stack(env, &old->stack[spi]);
19478		cur_reg = scalar_reg_for_stack(env, &cur->stack[spi]);
19479		if (old_reg && cur_reg) {
19480			if (!regsafe(env, old_reg, cur_reg, idmap, exact))
19481				return false;
19482			i += BPF_REG_SIZE - 1;
19483			continue;
19484		}
19485
19486		/* if old state was safe with misc data in the stack
19487		 * it will be safe with zero-initialized stack.
19488		 * The opposite is not true
19489		 */
19490		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC &&
19491		    cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO)
19492			continue;
19493		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] !=
19494		    cur->stack[spi].slot_type[i % BPF_REG_SIZE])
19495			/* Ex: old explored (safe) state has STACK_SPILL in
19496			 * this stack slot, but current has STACK_MISC ->
19497			 * this verifier states are not equivalent,
19498			 * return false to continue verification of this path
19499			 */
19500			return false;
19501		if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1)
19502			continue;
19503		/* Both old and cur are having same slot_type */
19504		switch (old->stack[spi].slot_type[BPF_REG_SIZE - 1]) {
19505		case STACK_SPILL:
19506			/* when explored and current stack slot are both storing
19507			 * spilled registers, check that stored pointers types
19508			 * are the same as well.
19509			 * Ex: explored safe path could have stored
19510			 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8}
19511			 * but current path has stored:
19512			 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16}
19513			 * such verifier states are not equivalent.
19514			 * return false to continue verification of this path
19515			 */
19516			if (!regsafe(env, &old->stack[spi].spilled_ptr,
19517				     &cur->stack[spi].spilled_ptr, idmap, exact))
19518				return false;
19519			break;
19520		case STACK_DYNPTR:
19521			old_reg = &old->stack[spi].spilled_ptr;
19522			cur_reg = &cur->stack[spi].spilled_ptr;
19523			if (old_reg->dynptr.type != cur_reg->dynptr.type ||
19524			    old_reg->dynptr.first_slot != cur_reg->dynptr.first_slot ||
19525			    !check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap))
19526				return false;
19527			break;
19528		case STACK_ITER:
19529			old_reg = &old->stack[spi].spilled_ptr;
19530			cur_reg = &cur->stack[spi].spilled_ptr;
19531			/* iter.depth is not compared between states as it
19532			 * doesn't matter for correctness and would otherwise
19533			 * prevent convergence; we maintain it only to prevent
19534			 * infinite loop check triggering, see
19535			 * iter_active_depths_differ()
19536			 */
19537			if (old_reg->iter.btf != cur_reg->iter.btf ||
19538			    old_reg->iter.btf_id != cur_reg->iter.btf_id ||
19539			    old_reg->iter.state != cur_reg->iter.state ||
19540			    /* ignore {old_reg,cur_reg}->iter.depth, see above */
19541			    !check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap))
19542				return false;
19543			break;
19544		case STACK_IRQ_FLAG:
19545			old_reg = &old->stack[spi].spilled_ptr;
19546			cur_reg = &cur->stack[spi].spilled_ptr;
19547			if (!check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap) ||
19548			    old_reg->irq.kfunc_class != cur_reg->irq.kfunc_class)
19549				return false;
19550			break;
19551		case STACK_MISC:
19552		case STACK_ZERO:
19553		case STACK_INVALID:
19554			continue;
19555		/* Ensure that new unhandled slot types return false by default */
19556		default:
19557			return false;
19558		}
19559	}
19560	return true;
19561}
19562
19563static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *cur,
19564		    struct bpf_idmap *idmap)
19565{
19566	int i;
19567
19568	if (old->acquired_refs != cur->acquired_refs)
19569		return false;
19570
19571	if (old->active_locks != cur->active_locks)
19572		return false;
19573
19574	if (old->active_preempt_locks != cur->active_preempt_locks)
19575		return false;
19576
19577	if (old->active_rcu_locks != cur->active_rcu_locks)
19578		return false;
19579
19580	if (!check_ids(old->active_irq_id, cur->active_irq_id, idmap))
19581		return false;
19582
19583	if (!check_ids(old->active_lock_id, cur->active_lock_id, idmap) ||
19584	    old->active_lock_ptr != cur->active_lock_ptr)
19585		return false;
19586
19587	for (i = 0; i < old->acquired_refs; i++) {
19588		if (!check_ids(old->refs[i].id, cur->refs[i].id, idmap) ||
19589		    old->refs[i].type != cur->refs[i].type)
19590			return false;
19591		switch (old->refs[i].type) {
19592		case REF_TYPE_PTR:
19593		case REF_TYPE_IRQ:
19594			break;
19595		case REF_TYPE_LOCK:
19596		case REF_TYPE_RES_LOCK:
19597		case REF_TYPE_RES_LOCK_IRQ:
19598			if (old->refs[i].ptr != cur->refs[i].ptr)
19599				return false;
19600			break;
19601		default:
19602			WARN_ONCE(1, "Unhandled enum type for reference state: %d\n", old->refs[i].type);
19603			return false;
19604		}
19605	}
19606
19607	return true;
19608}
19609
19610/* compare two verifier states
19611 *
19612 * all states stored in state_list are known to be valid, since
19613 * verifier reached 'bpf_exit' instruction through them
19614 *
19615 * this function is called when verifier exploring different branches of
19616 * execution popped from the state stack. If it sees an old state that has
19617 * more strict register state and more strict stack state then this execution
19618 * branch doesn't need to be explored further, since verifier already
19619 * concluded that more strict state leads to valid finish.
19620 *
19621 * Therefore two states are equivalent if register state is more conservative
19622 * and explored stack state is more conservative than the current one.
19623 * Example:
19624 *       explored                   current
19625 * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC)
19626 * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC)
19627 *
19628 * In other words if current stack state (one being explored) has more
19629 * valid slots than old one that already passed validation, it means
19630 * the verifier can stop exploring and conclude that current state is valid too
19631 *
19632 * Similarly with registers. If explored state has register type as invalid
19633 * whereas register type in current state is meaningful, it means that
19634 * the current state will reach 'bpf_exit' instruction safely
19635 */
19636static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old,
19637			      struct bpf_func_state *cur, u32 insn_idx, enum exact_level exact)
19638{
19639	u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before;
19640	u16 i;
19641
19642	if (old->callback_depth > cur->callback_depth)
19643		return false;
19644
19645	for (i = 0; i < MAX_BPF_REG; i++)
19646		if (((1 << i) & live_regs) &&
19647		    !regsafe(env, &old->regs[i], &cur->regs[i],
19648			     &env->idmap_scratch, exact))
19649			return false;
19650
19651	if (!stacksafe(env, old, cur, &env->idmap_scratch, exact))
19652		return false;
19653
19654	return true;
19655}
19656
19657static void reset_idmap_scratch(struct bpf_verifier_env *env)
19658{
19659	env->idmap_scratch.tmp_id_gen = env->id_gen;
19660	memset(&env->idmap_scratch.map, 0, sizeof(env->idmap_scratch.map));
19661}
19662
19663static bool states_equal(struct bpf_verifier_env *env,
19664			 struct bpf_verifier_state *old,
19665			 struct bpf_verifier_state *cur,
19666			 enum exact_level exact)
19667{
19668	u32 insn_idx;
19669	int i;
19670
19671	if (old->curframe != cur->curframe)
19672		return false;
19673
19674	reset_idmap_scratch(env);
19675
19676	/* Verification state from speculative execution simulation
19677	 * must never prune a non-speculative execution one.
19678	 */
19679	if (old->speculative && !cur->speculative)
19680		return false;
19681
19682	if (old->in_sleepable != cur->in_sleepable)
19683		return false;
19684
19685	if (!refsafe(old, cur, &env->idmap_scratch))
19686		return false;
19687
19688	/* for states to be equal callsites have to be the same
19689	 * and all frame states need to be equivalent
19690	 */
19691	for (i = 0; i <= old->curframe; i++) {
19692		insn_idx = frame_insn_idx(old, i);
19693		if (old->frame[i]->callsite != cur->frame[i]->callsite)
19694			return false;
19695		if (!func_states_equal(env, old->frame[i], cur->frame[i], insn_idx, exact))
19696			return false;
19697	}
19698	return true;
19699}
19700
19701/* find precise scalars in the previous equivalent state and
19702 * propagate them into the current state
19703 */
19704static int propagate_precision(struct bpf_verifier_env *env,
19705			       const struct bpf_verifier_state *old,
19706			       struct bpf_verifier_state *cur,
19707			       bool *changed)
19708{
19709	struct bpf_reg_state *state_reg;
19710	struct bpf_func_state *state;
19711	int i, err = 0, fr;
19712	bool first;
19713
19714	for (fr = old->curframe; fr >= 0; fr--) {
19715		state = old->frame[fr];
19716		state_reg = state->regs;
19717		first = true;
19718		for (i = 0; i < BPF_REG_FP; i++, state_reg++) {
19719			if (state_reg->type != SCALAR_VALUE ||
19720			    !state_reg->precise)
19721				continue;
19722			if (env->log.level & BPF_LOG_LEVEL2) {
19723				if (first)
19724					verbose(env, "frame %d: propagating r%d", fr, i);
19725				else
19726					verbose(env, ",r%d", i);
19727			}
19728			bt_set_frame_reg(&env->bt, fr, i);
19729			first = false;
19730		}
19731
19732		for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
19733			if (!is_spilled_reg(&state->stack[i]))
19734				continue;
19735			state_reg = &state->stack[i].spilled_ptr;
19736			if (state_reg->type != SCALAR_VALUE ||
19737			    !state_reg->precise)
19738				continue;
19739			if (env->log.level & BPF_LOG_LEVEL2) {
19740				if (first)
19741					verbose(env, "frame %d: propagating fp%d",
19742						fr, (-i - 1) * BPF_REG_SIZE);
19743				else
19744					verbose(env, ",fp%d", (-i - 1) * BPF_REG_SIZE);
19745			}
19746			bt_set_frame_slot(&env->bt, fr, i);
19747			first = false;
19748		}
19749		if (!first && (env->log.level & BPF_LOG_LEVEL2))
19750			verbose(env, "\n");
19751	}
19752
19753	err = __mark_chain_precision(env, cur, -1, changed);
19754	if (err < 0)
19755		return err;
19756
19757	return 0;
19758}
19759
19760#define MAX_BACKEDGE_ITERS 64
19761
19762/* Propagate read and precision marks from visit->backedges[*].state->equal_state
19763 * to corresponding parent states of visit->backedges[*].state until fixed point is reached,
19764 * then free visit->backedges.
19765 * After execution of this function incomplete_read_marks() will return false
19766 * for all states corresponding to @visit->callchain.
19767 */
19768static int propagate_backedges(struct bpf_verifier_env *env, struct bpf_scc_visit *visit)
19769{
19770	struct bpf_scc_backedge *backedge;
19771	struct bpf_verifier_state *st;
19772	bool changed;
19773	int i, err;
19774
19775	i = 0;
19776	do {
19777		if (i++ > MAX_BACKEDGE_ITERS) {
19778			if (env->log.level & BPF_LOG_LEVEL2)
19779				verbose(env, "%s: too many iterations\n", __func__);
19780			for (backedge = visit->backedges; backedge; backedge = backedge->next)
19781				mark_all_scalars_precise(env, &backedge->state);
19782			break;
19783		}
19784		changed = false;
19785		for (backedge = visit->backedges; backedge; backedge = backedge->next) {
19786			st = &backedge->state;
19787			err = propagate_precision(env, st->equal_state, st, &changed);
19788			if (err)
19789				return err;
19790		}
19791	} while (changed);
19792
19793	free_backedges(visit);
19794	return 0;
19795}
19796
19797static bool states_maybe_looping(struct bpf_verifier_state *old,
19798				 struct bpf_verifier_state *cur)
19799{
19800	struct bpf_func_state *fold, *fcur;
19801	int i, fr = cur->curframe;
19802
19803	if (old->curframe != fr)
19804		return false;
19805
19806	fold = old->frame[fr];
19807	fcur = cur->frame[fr];
19808	for (i = 0; i < MAX_BPF_REG; i++)
19809		if (memcmp(&fold->regs[i], &fcur->regs[i],
19810			   offsetof(struct bpf_reg_state, frameno)))
19811			return false;
19812	return true;
19813}
19814
19815static bool is_iter_next_insn(struct bpf_verifier_env *env, int insn_idx)
19816{
19817	return env->insn_aux_data[insn_idx].is_iter_next;
19818}
19819
19820/* is_state_visited() handles iter_next() (see process_iter_next_call() for
19821 * terminology) calls specially: as opposed to bounded BPF loops, it *expects*
19822 * states to match, which otherwise would look like an infinite loop. So while
19823 * iter_next() calls are taken care of, we still need to be careful and
19824 * prevent erroneous and too eager declaration of "infinite loop", when
19825 * iterators are involved.
19826 *
19827 * Here's a situation in pseudo-BPF assembly form:
19828 *
19829 *   0: again:                          ; set up iter_next() call args
19830 *   1:   r1 = &it                      ; <CHECKPOINT HERE>
19831 *   2:   call bpf_iter_num_next        ; this is iter_next() call
19832 *   3:   if r0 == 0 goto done
19833 *   4:   ... something useful here ...
19834 *   5:   goto again                    ; another iteration
19835 *   6: done:
19836 *   7:   r1 = &it
19837 *   8:   call bpf_iter_num_destroy     ; clean up iter state
19838 *   9:   exit
19839 *
19840 * This is a typical loop. Let's assume that we have a prune point at 1:,
19841 * before we get to `call bpf_iter_num_next` (e.g., because of that `goto
19842 * again`, assuming other heuristics don't get in a way).
19843 *
19844 * When we first time come to 1:, let's say we have some state X. We proceed
19845 * to 2:, fork states, enqueue ACTIVE, validate NULL case successfully, exit.
19846 * Now we come back to validate that forked ACTIVE state. We proceed through
19847 * 3-5, come to goto, jump to 1:. Let's assume our state didn't change, so we
19848 * are converging. But the problem is that we don't know that yet, as this
19849 * convergence has to happen at iter_next() call site only. So if nothing is
19850 * done, at 1: verifier will use bounded loop logic and declare infinite
19851 * looping (and would be *technically* correct, if not for iterator's
19852 * "eventual sticky NULL" contract, see process_iter_next_call()). But we
19853 * don't want that. So what we do in process_iter_next_call() when we go on
19854 * another ACTIVE iteration, we bump slot->iter.depth, to mark that it's
19855 * a different iteration. So when we suspect an infinite loop, we additionally
19856 * check if any of the *ACTIVE* iterator states depths differ. If yes, we
19857 * pretend we are not looping and wait for next iter_next() call.
19858 *
19859 * This only applies to ACTIVE state. In DRAINED state we don't expect to
19860 * loop, because that would actually mean infinite loop, as DRAINED state is
19861 * "sticky", and so we'll keep returning into the same instruction with the
19862 * same state (at least in one of possible code paths).
19863 *
19864 * This approach allows to keep infinite loop heuristic even in the face of
19865 * active iterator. E.g., C snippet below is and will be detected as
19866 * infinitely looping:
19867 *
19868 *   struct bpf_iter_num it;
19869 *   int *p, x;
19870 *
19871 *   bpf_iter_num_new(&it, 0, 10);
19872 *   while ((p = bpf_iter_num_next(&t))) {
19873 *       x = p;
19874 *       while (x--) {} // <<-- infinite loop here
19875 *   }
19876 *
19877 */
19878static bool iter_active_depths_differ(struct bpf_verifier_state *old, struct bpf_verifier_state *cur)
19879{
19880	struct bpf_reg_state *slot, *cur_slot;
19881	struct bpf_func_state *state;
19882	int i, fr;
19883
19884	for (fr = old->curframe; fr >= 0; fr--) {
19885		state = old->frame[fr];
19886		for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
19887			if (state->stack[i].slot_type[0] != STACK_ITER)
19888				continue;
19889
19890			slot = &state->stack[i].spilled_ptr;
19891			if (slot->iter.state != BPF_ITER_STATE_ACTIVE)
19892				continue;
19893
19894			cur_slot = &cur->frame[fr]->stack[i].spilled_ptr;
19895			if (cur_slot->iter.depth != slot->iter.depth)
19896				return true;
19897		}
19898	}
19899	return false;
19900}
19901
19902static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
19903{
19904	struct bpf_verifier_state_list *new_sl;
19905	struct bpf_verifier_state_list *sl;
19906	struct bpf_verifier_state *cur = env->cur_state, *new;
19907	bool force_new_state, add_new_state, loop;
19908	int n, err, states_cnt = 0;
19909	struct list_head *pos, *tmp, *head;
19910
19911	force_new_state = env->test_state_freq || is_force_checkpoint(env, insn_idx) ||
19912			  /* Avoid accumulating infinitely long jmp history */
19913			  cur->jmp_history_cnt > 40;
19914
19915	/* bpf progs typically have pruning point every 4 instructions
19916	 * http://vger.kernel.org/bpfconf2019.html#session-1
19917	 * Do not add new state for future pruning if the verifier hasn't seen
19918	 * at least 2 jumps and at least 8 instructions.
19919	 * This heuristics helps decrease 'total_states' and 'peak_states' metric.
19920	 * In tests that amounts to up to 50% reduction into total verifier
19921	 * memory consumption and 20% verifier time speedup.
19922	 */
19923	add_new_state = force_new_state;
19924	if (env->jmps_processed - env->prev_jmps_processed >= 2 &&
19925	    env->insn_processed - env->prev_insn_processed >= 8)
19926		add_new_state = true;
19927
19928	clean_live_states(env, insn_idx, cur);
19929
19930	loop = false;
19931	head = explored_state(env, insn_idx);
19932	list_for_each_safe(pos, tmp, head) {
19933		sl = container_of(pos, struct bpf_verifier_state_list, node);
19934		states_cnt++;
19935		if (sl->state.insn_idx != insn_idx)
19936			continue;
19937
19938		if (sl->state.branches) {
19939			struct bpf_func_state *frame = sl->state.frame[sl->state.curframe];
19940
19941			if (frame->in_async_callback_fn &&
19942			    frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) {
19943				/* Different async_entry_cnt means that the verifier is
19944				 * processing another entry into async callback.
19945				 * Seeing the same state is not an indication of infinite
19946				 * loop or infinite recursion.
19947				 * But finding the same state doesn't mean that it's safe
19948				 * to stop processing the current state. The previous state
19949				 * hasn't yet reached bpf_exit, since state.branches > 0.
19950				 * Checking in_async_callback_fn alone is not enough either.
19951				 * Since the verifier still needs to catch infinite loops
19952				 * inside async callbacks.
19953				 */
19954				goto skip_inf_loop_check;
19955			}
19956			/* BPF open-coded iterators loop detection is special.
19957			 * states_maybe_looping() logic is too simplistic in detecting
19958			 * states that *might* be equivalent, because it doesn't know
19959			 * about ID remapping, so don't even perform it.
19960			 * See process_iter_next_call() and iter_active_depths_differ()
19961			 * for overview of the logic. When current and one of parent
19962			 * states are detected as equivalent, it's a good thing: we prove
19963			 * convergence and can stop simulating further iterations.
19964			 * It's safe to assume that iterator loop will finish, taking into
19965			 * account iter_next() contract of eventually returning
19966			 * sticky NULL result.
19967			 *
19968			 * Note, that states have to be compared exactly in this case because
19969			 * read and precision marks might not be finalized inside the loop.
19970			 * E.g. as in the program below:
19971			 *
19972			 *     1. r7 = -16
19973			 *     2. r6 = bpf_get_prandom_u32()
19974			 *     3. while (bpf_iter_num_next(&fp[-8])) {
19975			 *     4.   if (r6 != 42) {
19976			 *     5.     r7 = -32
19977			 *     6.     r6 = bpf_get_prandom_u32()
19978			 *     7.     continue
19979			 *     8.   }
19980			 *     9.   r0 = r10
19981			 *    10.   r0 += r7
19982			 *    11.   r8 = *(u64 *)(r0 + 0)
19983			 *    12.   r6 = bpf_get_prandom_u32()
19984			 *    13. }
19985			 *
19986			 * Here verifier would first visit path 1-3, create a checkpoint at 3
19987			 * with r7=-16, continue to 4-7,3. Existing checkpoint at 3 does
19988			 * not have read or precision mark for r7 yet, thus inexact states
19989			 * comparison would discard current state with r7=-32
19990			 * => unsafe memory access at 11 would not be caught.
19991			 */
19992			if (is_iter_next_insn(env, insn_idx)) {
19993				if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) {
19994					struct bpf_func_state *cur_frame;
19995					struct bpf_reg_state *iter_state, *iter_reg;
19996					int spi;
19997
19998					cur_frame = cur->frame[cur->curframe];
19999					/* btf_check_iter_kfuncs() enforces that
20000					 * iter state pointer is always the first arg
20001					 */
20002					iter_reg = &cur_frame->regs[BPF_REG_1];
20003					/* current state is valid due to states_equal(),
20004					 * so we can assume valid iter and reg state,
20005					 * no need for extra (re-)validations
20006					 */
20007					spi = __get_spi(iter_reg->off + iter_reg->var_off.value);
20008					iter_state = &func(env, iter_reg)->stack[spi].spilled_ptr;
20009					if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE) {
20010						loop = true;
20011						goto hit;
20012					}
20013				}
20014				goto skip_inf_loop_check;
20015			}
20016			if (is_may_goto_insn_at(env, insn_idx)) {
20017				if (sl->state.may_goto_depth != cur->may_goto_depth &&
20018				    states_equal(env, &sl->state, cur, RANGE_WITHIN)) {
20019					loop = true;
20020					goto hit;
20021				}
20022			}
20023			if (bpf_calls_callback(env, insn_idx)) {
20024				if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) {
20025					loop = true;
20026					goto hit;
20027				}
20028				goto skip_inf_loop_check;
20029			}
20030			/* attempt to detect infinite loop to avoid unnecessary doomed work */
20031			if (states_maybe_looping(&sl->state, cur) &&
20032			    states_equal(env, &sl->state, cur, EXACT) &&
20033			    !iter_active_depths_differ(&sl->state, cur) &&
20034			    sl->state.may_goto_depth == cur->may_goto_depth &&
20035			    sl->state.callback_unroll_depth == cur->callback_unroll_depth) {
20036				verbose_linfo(env, insn_idx, "; ");
20037				verbose(env, "infinite loop detected at insn %d\n", insn_idx);
20038				verbose(env, "cur state:");
20039				print_verifier_state(env, cur, cur->curframe, true);
20040				verbose(env, "old state:");
20041				print_verifier_state(env, &sl->state, cur->curframe, true);
20042				return -EINVAL;
20043			}
20044			/* if the verifier is processing a loop, avoid adding new state
20045			 * too often, since different loop iterations have distinct
20046			 * states and may not help future pruning.
20047			 * This threshold shouldn't be too low to make sure that
20048			 * a loop with large bound will be rejected quickly.
20049			 * The most abusive loop will be:
20050			 * r1 += 1
20051			 * if r1 < 1000000 goto pc-2
20052			 * 1M insn_procssed limit / 100 == 10k peak states.
20053			 * This threshold shouldn't be too high either, since states
20054			 * at the end of the loop are likely to be useful in pruning.
20055			 */
20056skip_inf_loop_check:
20057			if (!force_new_state &&
20058			    env->jmps_processed - env->prev_jmps_processed < 20 &&
20059			    env->insn_processed - env->prev_insn_processed < 100)
20060				add_new_state = false;
20061			goto miss;
20062		}
20063		/* See comments for mark_all_regs_read_and_precise() */
20064		loop = incomplete_read_marks(env, &sl->state);
20065		if (states_equal(env, &sl->state, cur, loop ? RANGE_WITHIN : NOT_EXACT)) {
20066hit:
20067			sl->hit_cnt++;
20068
20069			/* if previous state reached the exit with precision and
20070			 * current state is equivalent to it (except precision marks)
20071			 * the precision needs to be propagated back in
20072			 * the current state.
20073			 */
20074			err = 0;
20075			if (is_jmp_point(env, env->insn_idx))
20076				err = push_jmp_history(env, cur, 0, 0);
20077			err = err ? : propagate_precision(env, &sl->state, cur, NULL);
20078			if (err)
20079				return err;
20080			/* When processing iterator based loops above propagate_liveness and
20081			 * propagate_precision calls are not sufficient to transfer all relevant
20082			 * read and precision marks. E.g. consider the following case:
20083			 *
20084			 *  .-> A --.  Assume the states are visited in the order A, B, C.
20085			 *  |   |   |  Assume that state B reaches a state equivalent to state A.
20086			 *  |   v   v  At this point, state C is not processed yet, so state A
20087			 *  '-- B   C  has not received any read or precision marks from C.
20088			 *             Thus, marks propagated from A to B are incomplete.
20089			 *
20090			 * The verifier mitigates this by performing the following steps:
20091			 *
20092			 * - Prior to the main verification pass, strongly connected components
20093			 *   (SCCs) are computed over the program's control flow graph,
20094			 *   intraprocedurally.
20095			 *
20096			 * - During the main verification pass, `maybe_enter_scc()` checks
20097			 *   whether the current verifier state is entering an SCC. If so, an
20098			 *   instance of a `bpf_scc_visit` object is created, and the state
20099			 *   entering the SCC is recorded as the entry state.
20100			 *
20101			 * - This instance is associated not with the SCC itself, but with a
20102			 *   `bpf_scc_callchain`: a tuple consisting of the call sites leading to
20103			 *   the SCC and the SCC id. See `compute_scc_callchain()`.
20104			 *
20105			 * - When a verification path encounters a `states_equal(...,
20106			 *   RANGE_WITHIN)` condition, there exists a call chain describing the
20107			 *   current state and a corresponding `bpf_scc_visit` instance. A copy
20108			 *   of the current state is created and added to
20109			 *   `bpf_scc_visit->backedges`.
20110			 *
20111			 * - When a verification path terminates, `maybe_exit_scc()` is called
20112			 *   from `update_branch_counts()`. For states with `branches == 0`, it
20113			 *   checks whether the state is the entry state of any `bpf_scc_visit`
20114			 *   instance. If it is, this indicates that all paths originating from
20115			 *   this SCC visit have been explored. `propagate_backedges()` is then
20116			 *   called, which propagates read and precision marks through the
20117			 *   backedges until a fixed point is reached.
20118			 *   (In the earlier example, this would propagate marks from A to B,
20119			 *    from C to A, and then again from A to B.)
20120			 *
20121			 * A note on callchains
20122			 * --------------------
20123			 *
20124			 * Consider the following example:
20125			 *
20126			 *     void foo() { loop { ... SCC#1 ... } }
20127			 *     void main() {
20128			 *       A: foo();
20129			 *       B: ...
20130			 *       C: foo();
20131			 *     }
20132			 *
20133			 * Here, there are two distinct callchains leading to SCC#1:
20134			 * - (A, SCC#1)
20135			 * - (C, SCC#1)
20136			 *
20137			 * Each callchain identifies a separate `bpf_scc_visit` instance that
20138			 * accumulates backedge states. The `propagate_{liveness,precision}()`
20139			 * functions traverse the parent state of each backedge state, which
20140			 * means these parent states must remain valid (i.e., not freed) while
20141			 * the corresponding `bpf_scc_visit` instance exists.
20142			 *
20143			 * Associating `bpf_scc_visit` instances directly with SCCs instead of
20144			 * callchains would break this invariant:
20145			 * - States explored during `C: foo()` would contribute backedges to
20146			 *   SCC#1, but SCC#1 would only be exited once the exploration of
20147			 *   `A: foo()` completes.
20148			 * - By that time, the states explored between `A: foo()` and `C: foo()`
20149			 *   (i.e., `B: ...`) may have already been freed, causing the parent
20150			 *   links for states from `C: foo()` to become invalid.
20151			 */
20152			if (loop) {
20153				struct bpf_scc_backedge *backedge;
20154
20155				backedge = kzalloc(sizeof(*backedge), GFP_KERNEL_ACCOUNT);
20156				if (!backedge)
20157					return -ENOMEM;
20158				err = copy_verifier_state(&backedge->state, cur);
20159				backedge->state.equal_state = &sl->state;
20160				backedge->state.insn_idx = insn_idx;
20161				err = err ?: add_scc_backedge(env, &sl->state, backedge);
20162				if (err) {
20163					free_verifier_state(&backedge->state, false);
20164					kfree(backedge);
20165					return err;
20166				}
20167			}
20168			return 1;
20169		}
20170miss:
20171		/* when new state is not going to be added do not increase miss count.
20172		 * Otherwise several loop iterations will remove the state
20173		 * recorded earlier. The goal of these heuristics is to have
20174		 * states from some iterations of the loop (some in the beginning
20175		 * and some at the end) to help pruning.
20176		 */
20177		if (add_new_state)
20178			sl->miss_cnt++;
20179		/* heuristic to determine whether this state is beneficial
20180		 * to keep checking from state equivalence point of view.
20181		 * Higher numbers increase max_states_per_insn and verification time,
20182		 * but do not meaningfully decrease insn_processed.
20183		 * 'n' controls how many times state could miss before eviction.
20184		 * Use bigger 'n' for checkpoints because evicting checkpoint states
20185		 * too early would hinder iterator convergence.
20186		 */
20187		n = is_force_checkpoint(env, insn_idx) && sl->state.branches > 0 ? 64 : 3;
20188		if (sl->miss_cnt > sl->hit_cnt * n + n) {
20189			/* the state is unlikely to be useful. Remove it to
20190			 * speed up verification
20191			 */
20192			sl->in_free_list = true;
20193			list_del(&sl->node);
20194			list_add(&sl->node, &env->free_list);
20195			env->free_list_size++;
20196			env->explored_states_size--;
20197			maybe_free_verifier_state(env, sl);
20198		}
20199	}
20200
20201	if (env->max_states_per_insn < states_cnt)
20202		env->max_states_per_insn = states_cnt;
20203
20204	if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES)
20205		return 0;
20206
20207	if (!add_new_state)
20208		return 0;
20209
20210	/* There were no equivalent states, remember the current one.
20211	 * Technically the current state is not proven to be safe yet,
20212	 * but it will either reach outer most bpf_exit (which means it's safe)
20213	 * or it will be rejected. When there are no loops the verifier won't be
20214	 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx)
20215	 * again on the way to bpf_exit.
20216	 * When looping the sl->state.branches will be > 0 and this state
20217	 * will not be considered for equivalence until branches == 0.
20218	 */
20219	new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL_ACCOUNT);
20220	if (!new_sl)
20221		return -ENOMEM;
20222	env->total_states++;
20223	env->explored_states_size++;
20224	update_peak_states(env);
20225	env->prev_jmps_processed = env->jmps_processed;
20226	env->prev_insn_processed = env->insn_processed;
20227
20228	/* forget precise markings we inherited, see __mark_chain_precision */
20229	if (env->bpf_capable)
20230		mark_all_scalars_imprecise(env, cur);
20231
20232	/* add new state to the head of linked list */
20233	new = &new_sl->state;
20234	err = copy_verifier_state(new, cur);
20235	if (err) {
20236		free_verifier_state(new, false);
20237		kfree(new_sl);
20238		return err;
20239	}
20240	new->insn_idx = insn_idx;
20241	verifier_bug_if(new->branches != 1, env,
20242			"%s:branches_to_explore=%d insn %d",
20243			__func__, new->branches, insn_idx);
20244	err = maybe_enter_scc(env, new);
20245	if (err) {
20246		free_verifier_state(new, false);
20247		kfree(new_sl);
20248		return err;
20249	}
20250
20251	cur->parent = new;
20252	cur->first_insn_idx = insn_idx;
20253	cur->dfs_depth = new->dfs_depth + 1;
20254	clear_jmp_history(cur);
20255	list_add(&new_sl->node, head);
20256	return 0;
20257}
20258
20259/* Return true if it's OK to have the same insn return a different type. */
20260static bool reg_type_mismatch_ok(enum bpf_reg_type type)
20261{
20262	switch (base_type(type)) {
20263	case PTR_TO_CTX:
20264	case PTR_TO_SOCKET:
20265	case PTR_TO_SOCK_COMMON:
20266	case PTR_TO_TCP_SOCK:
20267	case PTR_TO_XDP_SOCK:
20268	case PTR_TO_BTF_ID:
20269	case PTR_TO_ARENA:
20270		return false;
20271	default:
20272		return true;
20273	}
20274}
20275
20276/* If an instruction was previously used with particular pointer types, then we
20277 * need to be careful to avoid cases such as the below, where it may be ok
20278 * for one branch accessing the pointer, but not ok for the other branch:
20279 *
20280 * R1 = sock_ptr
20281 * goto X;
20282 * ...
20283 * R1 = some_other_valid_ptr;
20284 * goto X;
20285 * ...
20286 * R2 = *(u32 *)(R1 + 0);
20287 */
20288static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev)
20289{
20290	return src != prev && (!reg_type_mismatch_ok(src) ||
20291			       !reg_type_mismatch_ok(prev));
20292}
20293
20294static bool is_ptr_to_mem_or_btf_id(enum bpf_reg_type type)
20295{
20296	switch (base_type(type)) {
20297	case PTR_TO_MEM:
20298	case PTR_TO_BTF_ID:
20299		return true;
20300	default:
20301		return false;
20302	}
20303}
20304
20305static bool is_ptr_to_mem(enum bpf_reg_type type)
20306{
20307	return base_type(type) == PTR_TO_MEM;
20308}
20309
20310static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,
20311			     bool allow_trust_mismatch)
20312{
20313	enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type;
20314	enum bpf_reg_type merged_type;
20315
20316	if (*prev_type == NOT_INIT) {
20317		/* Saw a valid insn
20318		 * dst_reg = *(u32 *)(src_reg + off)
20319		 * save type to validate intersecting paths
20320		 */
20321		*prev_type = type;
20322	} else if (reg_type_mismatch(type, *prev_type)) {
20323		/* Abuser program is trying to use the same insn
20324		 * dst_reg = *(u32*) (src_reg + off)
20325		 * with different pointer types:
20326		 * src_reg == ctx in one branch and
20327		 * src_reg == stack|map in some other branch.
20328		 * Reject it.
20329		 */
20330		if (allow_trust_mismatch &&
20331		    is_ptr_to_mem_or_btf_id(type) &&
20332		    is_ptr_to_mem_or_btf_id(*prev_type)) {
20333			/*
20334			 * Have to support a use case when one path through
20335			 * the program yields TRUSTED pointer while another
20336			 * is UNTRUSTED. Fallback to UNTRUSTED to generate
20337			 * BPF_PROBE_MEM/BPF_PROBE_MEMSX.
20338			 * Same behavior of MEM_RDONLY flag.
20339			 */
20340			if (is_ptr_to_mem(type) || is_ptr_to_mem(*prev_type))
20341				merged_type = PTR_TO_MEM;
20342			else
20343				merged_type = PTR_TO_BTF_ID;
20344			if ((type & PTR_UNTRUSTED) || (*prev_type & PTR_UNTRUSTED))
20345				merged_type |= PTR_UNTRUSTED;
20346			if ((type & MEM_RDONLY) || (*prev_type & MEM_RDONLY))
20347				merged_type |= MEM_RDONLY;
20348			*prev_type = merged_type;
20349		} else {
20350			verbose(env, "same insn cannot be used with different pointers\n");
20351			return -EINVAL;
20352		}
20353	}
20354
20355	return 0;
20356}
20357
20358enum {
20359	PROCESS_BPF_EXIT = 1
20360};
20361
20362static int process_bpf_exit_full(struct bpf_verifier_env *env,
20363				 bool *do_print_state,
20364				 bool exception_exit)
20365{
20366	/* We must do check_reference_leak here before
20367	 * prepare_func_exit to handle the case when
20368	 * state->curframe > 0, it may be a callback function,
20369	 * for which reference_state must match caller reference
20370	 * state when it exits.
20371	 */
20372	int err = check_resource_leak(env, exception_exit,
20373				      exception_exit || !env->cur_state->curframe,
20374				      exception_exit ? "bpf_throw" :
20375				      "BPF_EXIT instruction in main prog");
20376	if (err)
20377		return err;
20378
20379	/* The side effect of the prepare_func_exit which is
20380	 * being skipped is that it frees bpf_func_state.
20381	 * Typically, process_bpf_exit will only be hit with
20382	 * outermost exit. copy_verifier_state in pop_stack will
20383	 * handle freeing of any extra bpf_func_state left over
20384	 * from not processing all nested function exits. We
20385	 * also skip return code checks as they are not needed
20386	 * for exceptional exits.
20387	 */
20388	if (exception_exit)
20389		return PROCESS_BPF_EXIT;
20390
20391	if (env->cur_state->curframe) {
20392		/* exit from nested function */
20393		err = prepare_func_exit(env, &env->insn_idx);
20394		if (err)
20395			return err;
20396		*do_print_state = true;
20397		return 0;
20398	}
20399
20400	err = check_return_code(env, BPF_REG_0, "R0");
20401	if (err)
20402		return err;
20403	return PROCESS_BPF_EXIT;
20404}
20405
20406static int indirect_jump_min_max_index(struct bpf_verifier_env *env,
20407				       int regno,
20408				       struct bpf_map *map,
20409				       u32 *pmin_index, u32 *pmax_index)
20410{
20411	struct bpf_reg_state *reg = reg_state(env, regno);
20412	u64 min_index, max_index;
20413	const u32 size = 8;
20414
20415	if (check_add_overflow(reg->umin_value, reg->off, &min_index) ||
20416		(min_index > (u64) U32_MAX * size)) {
20417		verbose(env, "the sum of R%u umin_value %llu and off %u is too big\n",
20418			     regno, reg->umin_value, reg->off);
20419		return -ERANGE;
20420	}
20421	if (check_add_overflow(reg->umax_value, reg->off, &max_index) ||
20422		(max_index > (u64) U32_MAX * size)) {
20423		verbose(env, "the sum of R%u umax_value %llu and off %u is too big\n",
20424			     regno, reg->umax_value, reg->off);
20425		return -ERANGE;
20426	}
20427
20428	min_index /= size;
20429	max_index /= size;
20430
20431	if (max_index >= map->max_entries) {
20432		verbose(env, "R%u points to outside of jump table: [%llu,%llu] max_entries %u\n",
20433			     regno, min_index, max_index, map->max_entries);
20434		return -EINVAL;
20435	}
20436
20437	*pmin_index = min_index;
20438	*pmax_index = max_index;
20439	return 0;
20440}
20441
20442/* gotox *dst_reg */
20443static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *insn)
20444{
20445	struct bpf_verifier_state *other_branch;
20446	struct bpf_reg_state *dst_reg;
20447	struct bpf_map *map;
20448	u32 min_index, max_index;
20449	int err = 0;
20450	int n;
20451	int i;
20452
20453	dst_reg = reg_state(env, insn->dst_reg);
20454	if (dst_reg->type != PTR_TO_INSN) {
20455		verbose(env, "R%d has type %s, expected PTR_TO_INSN\n",
20456			     insn->dst_reg, reg_type_str(env, dst_reg->type));
20457		return -EINVAL;
20458	}
20459
20460	map = dst_reg->map_ptr;
20461	if (verifier_bug_if(!map, env, "R%d has an empty map pointer", insn->dst_reg))
20462		return -EFAULT;
20463
20464	if (verifier_bug_if(map->map_type != BPF_MAP_TYPE_INSN_ARRAY, env,
20465			    "R%d has incorrect map type %d", insn->dst_reg, map->map_type))
20466		return -EFAULT;
20467
20468	err = indirect_jump_min_max_index(env, insn->dst_reg, map, &min_index, &max_index);
20469	if (err)
20470		return err;
20471
20472	/* Ensure that the buffer is large enough */
20473	if (!env->gotox_tmp_buf || env->gotox_tmp_buf->cnt < max_index - min_index + 1) {
20474		env->gotox_tmp_buf = iarray_realloc(env->gotox_tmp_buf,
20475						    max_index - min_index + 1);
20476		if (!env->gotox_tmp_buf)
20477			return -ENOMEM;
20478	}
20479
20480	n = copy_insn_array_uniq(map, min_index, max_index, env->gotox_tmp_buf->items);
20481	if (n < 0)
20482		return n;
20483	if (n == 0) {
20484		verbose(env, "register R%d doesn't point to any offset in map id=%d\n",
20485			     insn->dst_reg, map->id);
20486		return -EINVAL;
20487	}
20488
20489	for (i = 0; i < n - 1; i++) {
20490		other_branch = push_stack(env, env->gotox_tmp_buf->items[i],
20491					  env->insn_idx, env->cur_state->speculative);
20492		if (IS_ERR(other_branch))
20493			return PTR_ERR(other_branch);
20494	}
20495	env->insn_idx = env->gotox_tmp_buf->items[n-1];
20496	return 0;
20497}
20498
20499static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)
20500{
20501	int err;
20502	struct bpf_insn *insn = &env->prog->insnsi[env->insn_idx];
20503	u8 class = BPF_CLASS(insn->code);
20504
20505	if (class == BPF_ALU || class == BPF_ALU64) {
20506		err = check_alu_op(env, insn);
20507		if (err)
20508			return err;
20509
20510	} else if (class == BPF_LDX) {
20511		bool is_ldsx = BPF_MODE(insn->code) == BPF_MEMSX;
20512
20513		/* Check for reserved fields is already done in
20514		 * resolve_pseudo_ldimm64().
20515		 */
20516		err = check_load_mem(env, insn, false, is_ldsx, true, "ldx");
20517		if (err)
20518			return err;
20519	} else if (class == BPF_STX) {
20520		if (BPF_MODE(insn->code) == BPF_ATOMIC) {
20521			err = check_atomic(env, insn);
20522			if (err)
20523				return err;
20524			env->insn_idx++;
20525			return 0;
20526		}
20527
20528		if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) {
20529			verbose(env, "BPF_STX uses reserved fields\n");
20530			return -EINVAL;
20531		}
20532
20533		err = check_store_reg(env, insn, false);
20534		if (err)
20535			return err;
20536	} else if (class == BPF_ST) {
20537		enum bpf_reg_type dst_reg_type;
20538
20539		if (BPF_MODE(insn->code) != BPF_MEM ||
20540		    insn->src_reg != BPF_REG_0) {
20541			verbose(env, "BPF_ST uses reserved fields\n");
20542			return -EINVAL;
20543		}
20544		/* check src operand */
20545		err = check_reg_arg(env, insn->dst_reg, SRC_OP);
20546		if (err)
20547			return err;
20548
20549		dst_reg_type = cur_regs(env)[insn->dst_reg].type;
20550
20551		/* check that memory (dst_reg + off) is writeable */
20552		err = check_mem_access(env, env->insn_idx, insn->dst_reg,
20553				       insn->off, BPF_SIZE(insn->code),
20554				       BPF_WRITE, -1, false, false);
20555		if (err)
20556			return err;
20557
20558		err = save_aux_ptr_type(env, dst_reg_type, false);
20559		if (err)
20560			return err;
20561	} else if (class == BPF_JMP || class == BPF_JMP32) {
20562		u8 opcode = BPF_OP(insn->code);
20563
20564		env->jmps_processed++;
20565		if (opcode == BPF_CALL) {
20566			if (BPF_SRC(insn->code) != BPF_K ||
20567			    (insn->src_reg != BPF_PSEUDO_KFUNC_CALL &&
20568			     insn->off != 0) ||
20569			    (insn->src_reg != BPF_REG_0 &&
20570			     insn->src_reg != BPF_PSEUDO_CALL &&
20571			     insn->src_reg != BPF_PSEUDO_KFUNC_CALL) ||
20572			    insn->dst_reg != BPF_REG_0 || class == BPF_JMP32) {
20573				verbose(env, "BPF_CALL uses reserved fields\n");
20574				return -EINVAL;
20575			}
20576
20577			if (env->cur_state->active_locks) {
20578				if ((insn->src_reg == BPF_REG_0 &&
20579				     insn->imm != BPF_FUNC_spin_unlock) ||
20580				    (insn->src_reg == BPF_PSEUDO_KFUNC_CALL &&
20581				     (insn->off != 0 || !kfunc_spin_allowed(insn->imm)))) {
20582					verbose(env,
20583						"function calls are not allowed while holding a lock\n");
20584					return -EINVAL;
20585				}
20586			}
20587			if (insn->src_reg == BPF_PSEUDO_CALL) {
20588				err = check_func_call(env, insn, &env->insn_idx);
20589			} else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
20590				err = check_kfunc_call(env, insn, &env->insn_idx);
20591				if (!err && is_bpf_throw_kfunc(insn))
20592					return process_bpf_exit_full(env, do_print_state, true);
20593			} else {
20594				err = check_helper_call(env, insn, &env->insn_idx);
20595			}
20596			if (err)
20597				return err;
20598
20599			mark_reg_scratched(env, BPF_REG_0);
20600		} else if (opcode == BPF_JA) {
20601			if (BPF_SRC(insn->code) == BPF_X) {
20602				if (insn->src_reg != BPF_REG_0 ||
20603				    insn->imm != 0 || insn->off != 0) {
20604					verbose(env, "BPF_JA|BPF_X uses reserved fields\n");
20605					return -EINVAL;
20606				}
20607				return check_indirect_jump(env, insn);
20608			}
20609
20610			if (BPF_SRC(insn->code) != BPF_K ||
20611			    insn->src_reg != BPF_REG_0 ||
20612			    insn->dst_reg != BPF_REG_0 ||
20613			    (class == BPF_JMP && insn->imm != 0) ||
20614			    (class == BPF_JMP32 && insn->off != 0)) {
20615				verbose(env, "BPF_JA uses reserved fields\n");
20616				return -EINVAL;
20617			}
20618
20619			if (class == BPF_JMP)
20620				env->insn_idx += insn->off + 1;
20621			else
20622				env->insn_idx += insn->imm + 1;
20623			return 0;
20624		} else if (opcode == BPF_EXIT) {
20625			if (BPF_SRC(insn->code) != BPF_K ||
20626			    insn->imm != 0 ||
20627			    insn->src_reg != BPF_REG_0 ||
20628			    insn->dst_reg != BPF_REG_0 ||
20629			    class == BPF_JMP32) {
20630				verbose(env, "BPF_EXIT uses reserved fields\n");
20631				return -EINVAL;
20632			}
20633			return process_bpf_exit_full(env, do_print_state, false);
20634		} else {
20635			err = check_cond_jmp_op(env, insn, &env->insn_idx);
20636			if (err)
20637				return err;
20638		}
20639	} else if (class == BPF_LD) {
20640		u8 mode = BPF_MODE(insn->code);
20641
20642		if (mode == BPF_ABS || mode == BPF_IND) {
20643			err = check_ld_abs(env, insn);
20644			if (err)
20645				return err;
20646
20647		} else if (mode == BPF_IMM) {
20648			err = check_ld_imm(env, insn);
20649			if (err)
20650				return err;
20651
20652			env->insn_idx++;
20653			sanitize_mark_insn_seen(env);
20654		} else {
20655			verbose(env, "invalid BPF_LD mode\n");
20656			return -EINVAL;
20657		}
20658	} else {
20659		verbose(env, "unknown insn class %d\n", class);
20660		return -EINVAL;
20661	}
20662
20663	env->insn_idx++;
20664	return 0;
20665}
20666
20667static int do_check(struct bpf_verifier_env *env)
20668{
20669	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
20670	struct bpf_verifier_state *state = env->cur_state;
20671	struct bpf_insn *insns = env->prog->insnsi;
20672	int insn_cnt = env->prog->len;
20673	bool do_print_state = false;
20674	int prev_insn_idx = -1;
20675
20676	for (;;) {
20677		struct bpf_insn *insn;
20678		struct bpf_insn_aux_data *insn_aux;
20679		int err, marks_err;
20680
20681		/* reset current history entry on each new instruction */
20682		env->cur_hist_ent = NULL;
20683
20684		env->prev_insn_idx = prev_insn_idx;
20685		if (env->insn_idx >= insn_cnt) {
20686			verbose(env, "invalid insn idx %d insn_cnt %d\n",
20687				env->insn_idx, insn_cnt);
20688			return -EFAULT;
20689		}
20690
20691		insn = &insns[env->insn_idx];
20692		insn_aux = &env->insn_aux_data[env->insn_idx];
20693
20694		if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {
20695			verbose(env,
20696				"BPF program is too large. Processed %d insn\n",
20697				env->insn_processed);
20698			return -E2BIG;
20699		}
20700
20701		state->last_insn_idx = env->prev_insn_idx;
20702		state->insn_idx = env->insn_idx;
20703
20704		if (is_prune_point(env, env->insn_idx)) {
20705			err = is_state_visited(env, env->insn_idx);
20706			if (err < 0)
20707				return err;
20708			if (err == 1) {
20709				/* found equivalent state, can prune the search */
20710				if (env->log.level & BPF_LOG_LEVEL) {
20711					if (do_print_state)
20712						verbose(env, "\nfrom %d to %d%s: safe\n",
20713							env->prev_insn_idx, env->insn_idx,
20714							env->cur_state->speculative ?
20715							" (speculative execution)" : "");
20716					else
20717						verbose(env, "%d: safe\n", env->insn_idx);
20718				}
20719				goto process_bpf_exit;
20720			}
20721		}
20722
20723		if (is_jmp_point(env, env->insn_idx)) {
20724			err = push_jmp_history(env, state, 0, 0);
20725			if (err)
20726				return err;
20727		}
20728
20729		if (signal_pending(current))
20730			return -EAGAIN;
20731
20732		if (need_resched())
20733			cond_resched();
20734
20735		if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) {
20736			verbose(env, "\nfrom %d to %d%s:",
20737				env->prev_insn_idx, env->insn_idx,
20738				env->cur_state->speculative ?
20739				" (speculative execution)" : "");
20740			print_verifier_state(env, state, state->curframe, true);
20741			do_print_state = false;
20742		}
20743
20744		if (env->log.level & BPF_LOG_LEVEL) {
20745			if (verifier_state_scratched(env))
20746				print_insn_state(env, state, state->curframe);
20747
20748			verbose_linfo(env, env->insn_idx, "; ");
20749			env->prev_log_pos = env->log.end_pos;
20750			verbose(env, "%d: ", env->insn_idx);
20751			verbose_insn(env, insn);
20752			env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos;
20753			env->prev_log_pos = env->log.end_pos;
20754		}
20755
20756		if (bpf_prog_is_offloaded(env->prog->aux)) {
20757			err = bpf_prog_offload_verify_insn(env, env->insn_idx,
20758							   env->prev_insn_idx);
20759			if (err)
20760				return err;
20761		}
20762
20763		sanitize_mark_insn_seen(env);
20764		prev_insn_idx = env->insn_idx;
20765
20766		/* Reduce verification complexity by stopping speculative path
20767		 * verification when a nospec is encountered.
20768		 */
20769		if (state->speculative && insn_aux->nospec)
20770			goto process_bpf_exit;
20771
20772		err = bpf_reset_stack_write_marks(env, env->insn_idx);
20773		if (err)
20774			return err;
20775		err = do_check_insn(env, &do_print_state);
20776		if (err >= 0 || error_recoverable_with_nospec(err)) {
20777			marks_err = bpf_commit_stack_write_marks(env);
20778			if (marks_err)
20779				return marks_err;
20780		}
20781		if (error_recoverable_with_nospec(err) && state->speculative) {
20782			/* Prevent this speculative path from ever reaching the
20783			 * insn that would have been unsafe to execute.
20784			 */
20785			insn_aux->nospec = true;
20786			/* If it was an ADD/SUB insn, potentially remove any
20787			 * markings for alu sanitization.
20788			 */
20789			insn_aux->alu_state = 0;
20790			goto process_bpf_exit;
20791		} else if (err < 0) {
20792			return err;
20793		} else if (err == PROCESS_BPF_EXIT) {
20794			goto process_bpf_exit;
20795		}
20796		WARN_ON_ONCE(err);
20797
20798		if (state->speculative && insn_aux->nospec_result) {
20799			/* If we are on a path that performed a jump-op, this
20800			 * may skip a nospec patched-in after the jump. This can
20801			 * currently never happen because nospec_result is only
20802			 * used for the write-ops
20803			 * `*(size*)(dst_reg+off)=src_reg|imm32` and helper
20804			 * calls. These must never skip the following insn
20805			 * (i.e., bpf_insn_successors()'s opcode_info.can_jump
20806			 * is false). Still, add a warning to document this in
20807			 * case nospec_result is used elsewhere in the future.
20808			 *
20809			 * All non-branch instructions have a single
20810			 * fall-through edge. For these, nospec_result should
20811			 * already work.
20812			 */
20813			if (verifier_bug_if((BPF_CLASS(insn->code) == BPF_JMP ||
20814					     BPF_CLASS(insn->code) == BPF_JMP32) &&
20815					    BPF_OP(insn->code) != BPF_CALL, env,
20816					    "speculation barrier after jump instruction may not have the desired effect"))
20817				return -EFAULT;
20818process_bpf_exit:
20819			mark_verifier_state_scratched(env);
20820			err = update_branch_counts(env, env->cur_state);
20821			if (err)
20822				return err;
20823			err = bpf_update_live_stack(env);
20824			if (err)
20825				return err;
20826			err = pop_stack(env, &prev_insn_idx, &env->insn_idx,
20827					pop_log);
20828			if (err < 0) {
20829				if (err != -ENOENT)
20830					return err;
20831				break;
20832			} else {
20833				do_print_state = true;
20834				continue;
20835			}
20836		}
20837	}
20838
20839	return 0;
20840}
20841
20842static int find_btf_percpu_datasec(struct btf *btf)
20843{
20844	const struct btf_type *t;
20845	const char *tname;
20846	int i, n;
20847
20848	/*
20849	 * Both vmlinux and module each have their own ".data..percpu"
20850	 * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF
20851	 * types to look at only module's own BTF types.
20852	 */
20853	n = btf_nr_types(btf);
20854	if (btf_is_module(btf))
20855		i = btf_nr_types(btf_vmlinux);
20856	else
20857		i = 1;
20858
20859	for(; i < n; i++) {
20860		t = btf_type_by_id(btf, i);
20861		if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC)
20862			continue;
20863
20864		tname = btf_name_by_offset(btf, t->name_off);
20865		if (!strcmp(tname, ".data..percpu"))
20866			return i;
20867	}
20868
20869	return -ENOENT;
20870}
20871
20872/*
20873 * Add btf to the env->used_btfs array. If needed, refcount the
20874 * corresponding kernel module. To simplify caller's logic
20875 * in case of error or if btf was added before the function
20876 * decreases the btf refcount.
20877 */
20878static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf)
20879{
20880	struct btf_mod_pair *btf_mod;
20881	int ret = 0;
20882	int i;
20883
20884	/* check whether we recorded this BTF (and maybe module) already */
20885	for (i = 0; i < env->used_btf_cnt; i++)
20886		if (env->used_btfs[i].btf == btf)
20887			goto ret_put;
20888
20889	if (env->used_btf_cnt >= MAX_USED_BTFS) {
20890		verbose(env, "The total number of btfs per program has reached the limit of %u\n",
20891			MAX_USED_BTFS);
20892		ret = -E2BIG;
20893		goto ret_put;
20894	}
20895
20896	btf_mod = &env->used_btfs[env->used_btf_cnt];
20897	btf_mod->btf = btf;
20898	btf_mod->module = NULL;
20899
20900	/* if we reference variables from kernel module, bump its refcount */
20901	if (btf_is_module(btf)) {
20902		btf_mod->module = btf_try_get_module(btf);
20903		if (!btf_mod->module) {
20904			ret = -ENXIO;
20905			goto ret_put;
20906		}
20907	}
20908
20909	env->used_btf_cnt++;
20910	return 0;
20911
20912ret_put:
20913	/* Either error or this BTF was already added */
20914	btf_put(btf);
20915	return ret;
20916}
20917
20918/* replace pseudo btf_id with kernel symbol address */
20919static int __check_pseudo_btf_id(struct bpf_verifier_env *env,
20920				 struct bpf_insn *insn,
20921				 struct bpf_insn_aux_data *aux,
20922				 struct btf *btf)
20923{
20924	const struct btf_var_secinfo *vsi;
20925	const struct btf_type *datasec;
20926	const struct btf_type *t;
20927	const char *sym_name;
20928	bool percpu = false;
20929	u32 type, id = insn->imm;
20930	s32 datasec_id;
20931	u64 addr;
20932	int i;
20933
20934	t = btf_type_by_id(btf, id);
20935	if (!t) {
20936		verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);
20937		return -ENOENT;
20938	}
20939
20940	if (!btf_type_is_var(t) && !btf_type_is_func(t)) {
20941		verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR or KIND_FUNC\n", id);
20942		return -EINVAL;
20943	}
20944
20945	sym_name = btf_name_by_offset(btf, t->name_off);
20946	addr = kallsyms_lookup_name(sym_name);
20947	if (!addr) {
20948		verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n",
20949			sym_name);
20950		return -ENOENT;
20951	}
20952	insn[0].imm = (u32)addr;
20953	insn[1].imm = addr >> 32;
20954
20955	if (btf_type_is_func(t)) {
20956		aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY;
20957		aux->btf_var.mem_size = 0;
20958		return 0;
20959	}
20960
20961	datasec_id = find_btf_percpu_datasec(btf);
20962	if (datasec_id > 0) {
20963		datasec = btf_type_by_id(btf, datasec_id);
20964		for_each_vsi(i, datasec, vsi) {
20965			if (vsi->type == id) {
20966				percpu = true;
20967				break;
20968			}
20969		}
20970	}
20971
20972	type = t->type;
20973	t = btf_type_skip_modifiers(btf, type, NULL);
20974	if (percpu) {
20975		aux->btf_var.reg_type = PTR_TO_BTF_ID | MEM_PERCPU;
20976		aux->btf_var.btf = btf;
20977		aux->btf_var.btf_id = type;
20978	} else if (!btf_type_is_struct(t)) {
20979		const struct btf_type *ret;
20980		const char *tname;
20981		u32 tsize;
20982
20983		/* resolve the type size of ksym. */
20984		ret = btf_resolve_size(btf, t, &tsize);
20985		if (IS_ERR(ret)) {
20986			tname = btf_name_by_offset(btf, t->name_off);
20987			verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n",
20988				tname, PTR_ERR(ret));
20989			return -EINVAL;
20990		}
20991		aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY;
20992		aux->btf_var.mem_size = tsize;
20993	} else {
20994		aux->btf_var.reg_type = PTR_TO_BTF_ID;
20995		aux->btf_var.btf = btf;
20996		aux->btf_var.btf_id = type;
20997	}
20998
20999	return 0;
21000}
21001
21002static int check_pseudo_btf_id(struct bpf_verifier_env *env,
21003			       struct bpf_insn *insn,
21004			       struct bpf_insn_aux_data *aux)
21005{
21006	struct btf *btf;
21007	int btf_fd;
21008	int err;
21009
21010	btf_fd = insn[1].imm;
21011	if (btf_fd) {
21012		btf = btf_get_by_fd(btf_fd);
21013		if (IS_ERR(btf)) {
21014			verbose(env, "invalid module BTF object FD specified.\n");
21015			return -EINVAL;
21016		}
21017	} else {
21018		if (!btf_vmlinux) {
21019			verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");
21020			return -EINVAL;
21021		}
21022		btf_get(btf_vmlinux);
21023		btf = btf_vmlinux;
21024	}
21025
21026	err = __check_pseudo_btf_id(env, insn, aux, btf);
21027	if (err) {
21028		btf_put(btf);
21029		return err;
21030	}
21031
21032	return __add_used_btf(env, btf);
21033}
21034
21035static bool is_tracing_prog_type(enum bpf_prog_type type)
21036{
21037	switch (type) {
21038	case BPF_PROG_TYPE_KPROBE:
21039	case BPF_PROG_TYPE_TRACEPOINT:
21040	case BPF_PROG_TYPE_PERF_EVENT:
21041	case BPF_PROG_TYPE_RAW_TRACEPOINT:
21042	case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
21043		return true;
21044	default:
21045		return false;
21046	}
21047}
21048
21049static bool bpf_map_is_cgroup_storage(struct bpf_map *map)
21050{
21051	return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE ||
21052		map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);
21053}
21054
21055static int check_map_prog_compatibility(struct bpf_verifier_env *env,
21056					struct bpf_map *map,
21057					struct bpf_prog *prog)
21058
21059{
21060	enum bpf_prog_type prog_type = resolve_prog_type(prog);
21061
21062	if (map->excl_prog_sha &&
21063	    memcmp(map->excl_prog_sha, prog->digest, SHA256_DIGEST_SIZE)) {
21064		verbose(env, "program's hash doesn't match map's excl_prog_hash\n");
21065		return -EACCES;
21066	}
21067
21068	if (btf_record_has_field(map->record, BPF_LIST_HEAD) ||
21069	    btf_record_has_field(map->record, BPF_RB_ROOT)) {
21070		if (is_tracing_prog_type(prog_type)) {
21071			verbose(env, "tracing progs cannot use bpf_{list_head,rb_root} yet\n");
21072			return -EINVAL;
21073		}
21074	}
21075
21076	if (btf_record_has_field(map->record, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK)) {
21077		if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) {
21078			verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n");
21079			return -EINVAL;
21080		}
21081
21082		if (is_tracing_prog_type(prog_type)) {
21083			verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
21084			return -EINVAL;
21085		}
21086	}
21087
21088	if (btf_record_has_field(map->record, BPF_TIMER)) {
21089		if (is_tracing_prog_type(prog_type)) {
21090			verbose(env, "tracing progs cannot use bpf_timer yet\n");
21091			return -EINVAL;
21092		}
21093	}
21094
21095	if (btf_record_has_field(map->record, BPF_WORKQUEUE)) {
21096		if (is_tracing_prog_type(prog_type)) {
21097			verbose(env, "tracing progs cannot use bpf_wq yet\n");
21098			return -EINVAL;
21099		}
21100	}
21101
21102	if ((bpf_prog_is_offloaded(prog->aux) || bpf_map_is_offloaded(map)) &&
21103	    !bpf_offload_prog_map_match(prog, map)) {
21104		verbose(env, "offload device mismatch between prog and map\n");
21105		return -EINVAL;
21106	}
21107
21108	if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
21109		verbose(env, "bpf_struct_ops map cannot be used in prog\n");
21110		return -EINVAL;
21111	}
21112
21113	if (prog->sleepable)
21114		switch (map->map_type) {
21115		case BPF_MAP_TYPE_HASH:
21116		case BPF_MAP_TYPE_LRU_HASH:
21117		case BPF_MAP_TYPE_ARRAY:
21118		case BPF_MAP_TYPE_PERCPU_HASH:
21119		case BPF_MAP_TYPE_PERCPU_ARRAY:
21120		case BPF_MAP_TYPE_LRU_PERCPU_HASH:
21121		case BPF_MAP_TYPE_ARRAY_OF_MAPS:
21122		case BPF_MAP_TYPE_HASH_OF_MAPS:
21123		case BPF_MAP_TYPE_RINGBUF:
21124		case BPF_MAP_TYPE_USER_RINGBUF:
21125		case BPF_MAP_TYPE_INODE_STORAGE:
21126		case BPF_MAP_TYPE_SK_STORAGE:
21127		case BPF_MAP_TYPE_TASK_STORAGE:
21128		case BPF_MAP_TYPE_CGRP_STORAGE:
21129		case BPF_MAP_TYPE_QUEUE:
21130		case BPF_MAP_TYPE_STACK:
21131		case BPF_MAP_TYPE_ARENA:
21132		case BPF_MAP_TYPE_INSN_ARRAY:
21133			break;
21134		default:
21135			verbose(env,
21136				"Sleepable programs can only use array, hash, ringbuf and local storage maps\n");
21137			return -EINVAL;
21138		}
21139
21140	if (bpf_map_is_cgroup_storage(map) &&
21141	    bpf_cgroup_storage_assign(env->prog->aux, map)) {
21142		verbose(env, "only one cgroup storage of each type is allowed\n");
21143		return -EBUSY;
21144	}
21145
21146	if (map->map_type == BPF_MAP_TYPE_ARENA) {
21147		if (env->prog->aux->arena) {
21148			verbose(env, "Only one arena per program\n");
21149			return -EBUSY;
21150		}
21151		if (!env->allow_ptr_leaks || !env->bpf_capable) {
21152			verbose(env, "CAP_BPF and CAP_PERFMON are required to use arena\n");
21153			return -EPERM;
21154		}
21155		if (!env->prog->jit_requested) {
21156			verbose(env, "JIT is required to use arena\n");
21157			return -EOPNOTSUPP;
21158		}
21159		if (!bpf_jit_supports_arena()) {
21160			verbose(env, "JIT doesn't support arena\n");
21161			return -EOPNOTSUPP;
21162		}
21163		env->prog->aux->arena = (void *)map;
21164		if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) {
21165			verbose(env, "arena's user address must be set via map_extra or mmap()\n");
21166			return -EINVAL;
21167		}
21168	}
21169
21170	return 0;
21171}
21172
21173static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map)
21174{
21175	int i, err;
21176
21177	/* check whether we recorded this map already */
21178	for (i = 0; i < env->used_map_cnt; i++)
21179		if (env->used_maps[i] == map)
21180			return i;
21181
21182	if (env->used_map_cnt >= MAX_USED_MAPS) {
21183		verbose(env, "The total number of maps per program has reached the limit of %u\n",
21184			MAX_USED_MAPS);
21185		return -E2BIG;
21186	}
21187
21188	err = check_map_prog_compatibility(env, map, env->prog);
21189	if (err)
21190		return err;
21191
21192	if (env->prog->sleepable)
21193		atomic64_inc(&map->sleepable_refcnt);
21194
21195	/* hold the map. If the program is rejected by verifier,
21196	 * the map will be released by release_maps() or it
21197	 * will be used by the valid program until it's unloaded
21198	 * and all maps are released in bpf_free_used_maps()
21199	 */
21200	bpf_map_inc(map);
21201
21202	env->used_maps[env->used_map_cnt++] = map;
21203
21204	if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) {
21205		err = bpf_insn_array_init(map, env->prog);
21206		if (err) {
21207			verbose(env, "Failed to properly initialize insn array\n");
21208			return err;
21209		}
21210		env->insn_array_maps[env->insn_array_map_cnt++] = map;
21211	}
21212
21213	return env->used_map_cnt - 1;
21214}
21215
21216/* Add map behind fd to used maps list, if it's not already there, and return
21217 * its index.
21218 * Returns <0 on error, or >= 0 index, on success.
21219 */
21220static int add_used_map(struct bpf_verifier_env *env, int fd)
21221{
21222	struct bpf_map *map;
21223	CLASS(fd, f)(fd);
21224
21225	map = __bpf_map_get(f);
21226	if (IS_ERR(map)) {
21227		verbose(env, "fd %d is not pointing to valid bpf_map\n", fd);
21228		return PTR_ERR(map);
21229	}
21230
21231	return __add_used_map(env, map);
21232}
21233
21234/* find and rewrite pseudo imm in ld_imm64 instructions:
21235 *
21236 * 1. if it accesses map FD, replace it with actual map pointer.
21237 * 2. if it accesses btf_id of a VAR, replace it with pointer to the var.
21238 *
21239 * NOTE: btf_vmlinux is required for converting pseudo btf_id.
21240 */
21241static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)
21242{
21243	struct bpf_insn *insn = env->prog->insnsi;
21244	int insn_cnt = env->prog->len;
21245	int i, err;
21246
21247	err = bpf_prog_calc_tag(env->prog);
21248	if (err)
21249		return err;
21250
21251	for (i = 0; i < insn_cnt; i++, insn++) {
21252		if (BPF_CLASS(insn->code) == BPF_LDX &&
21253		    ((BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) ||
21254		    insn->imm != 0)) {
21255			verbose(env, "BPF_LDX uses reserved fields\n");
21256			return -EINVAL;
21257		}
21258
21259		if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {
21260			struct bpf_insn_aux_data *aux;
21261			struct bpf_map *map;
21262			int map_idx;
21263			u64 addr;
21264			u32 fd;
21265
21266			if (i == insn_cnt - 1 || insn[1].code != 0 ||
21267			    insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||
21268			    insn[1].off != 0) {
21269				verbose(env, "invalid bpf_ld_imm64 insn\n");
21270				return -EINVAL;
21271			}
21272
21273			if (insn[0].src_reg == 0)
21274				/* valid generic load 64-bit imm */
21275				goto next_insn;
21276
21277			if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) {
21278				aux = &env->insn_aux_data[i];
21279				err = check_pseudo_btf_id(env, insn, aux);
21280				if (err)
21281					return err;
21282				goto next_insn;
21283			}
21284
21285			if (insn[0].src_reg == BPF_PSEUDO_FUNC) {
21286				aux = &env->insn_aux_data[i];
21287				aux->ptr_type = PTR_TO_FUNC;
21288				goto next_insn;
21289			}
21290
21291			/* In final convert_pseudo_ld_imm64() step, this is
21292			 * converted into regular 64-bit imm load insn.
21293			 */
21294			switch (insn[0].src_reg) {
21295			case BPF_PSEUDO_MAP_VALUE:
21296			case BPF_PSEUDO_MAP_IDX_VALUE:
21297				break;
21298			case BPF_PSEUDO_MAP_FD:
21299			case BPF_PSEUDO_MAP_IDX:
21300				if (insn[1].imm == 0)
21301					break;
21302				fallthrough;
21303			default:
21304				verbose(env, "unrecognized bpf_ld_imm64 insn\n");
21305				return -EINVAL;
21306			}
21307
21308			switch (insn[0].src_reg) {
21309			case BPF_PSEUDO_MAP_IDX_VALUE:
21310			case BPF_PSEUDO_MAP_IDX:
21311				if (bpfptr_is_null(env->fd_array)) {
21312					verbose(env, "fd_idx without fd_array is invalid\n");
21313					return -EPROTO;
21314				}
21315				if (copy_from_bpfptr_offset(&fd, env->fd_array,
21316							    insn[0].imm * sizeof(fd),
21317							    sizeof(fd)))
21318					return -EFAULT;
21319				break;
21320			default:
21321				fd = insn[0].imm;
21322				break;
21323			}
21324
21325			map_idx = add_used_map(env, fd);
21326			if (map_idx < 0)
21327				return map_idx;
21328			map = env->used_maps[map_idx];
21329
21330			aux = &env->insn_aux_data[i];
21331			aux->map_index = map_idx;
21332
21333			if (insn[0].src_reg == BPF_PSEUDO_MAP_FD ||
21334			    insn[0].src_reg == BPF_PSEUDO_MAP_IDX) {
21335				addr = (unsigned long)map;
21336			} else {
21337				u32 off = insn[1].imm;
21338
21339				if (off >= BPF_MAX_VAR_OFF) {
21340					verbose(env, "direct value offset of %u is not allowed\n", off);
21341					return -EINVAL;
21342				}
21343
21344				if (!map->ops->map_direct_value_addr) {
21345					verbose(env, "no direct value access support for this map type\n");
21346					return -EINVAL;
21347				}
21348
21349				err = map->ops->map_direct_value_addr(map, &addr, off);
21350				if (err) {
21351					verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n",
21352						map->value_size, off);
21353					return err;
21354				}
21355
21356				aux->map_off = off;
21357				addr += off;
21358			}
21359
21360			insn[0].imm = (u32)addr;
21361			insn[1].imm = addr >> 32;
21362
21363next_insn:
21364			insn++;
21365			i++;
21366			continue;
21367		}
21368
21369		/* Basic sanity check before we invest more work here. */
21370		if (!bpf_opcode_in_insntable(insn->code)) {
21371			verbose(env, "unknown opcode %02x\n", insn->code);
21372			return -EINVAL;
21373		}
21374	}
21375
21376	/* now all pseudo BPF_LD_IMM64 instructions load valid
21377	 * 'struct bpf_map *' into a register instead of user map_fd.
21378	 * These pointers will be used later by verifier to validate map access.
21379	 */
21380	return 0;
21381}
21382
21383/* drop refcnt of maps used by the rejected program */
21384static void release_maps(struct bpf_verifier_env *env)
21385{
21386	__bpf_free_used_maps(env->prog->aux, env->used_maps,
21387			     env->used_map_cnt);
21388}
21389
21390/* drop refcnt of maps used by the rejected program */
21391static void release_btfs(struct bpf_verifier_env *env)
21392{
21393	__bpf_free_used_btfs(env->used_btfs, env->used_btf_cnt);
21394}
21395
21396/* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */
21397static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)
21398{
21399	struct bpf_insn *insn = env->prog->insnsi;
21400	int insn_cnt = env->prog->len;
21401	int i;
21402
21403	for (i = 0; i < insn_cnt; i++, insn++) {
21404		if (insn->code != (BPF_LD | BPF_IMM | BPF_DW))
21405			continue;
21406		if (insn->src_reg == BPF_PSEUDO_FUNC)
21407			continue;
21408		insn->src_reg = 0;
21409	}
21410}
21411
21412/* single env->prog->insni[off] instruction was replaced with the range
21413 * insni[off, off + cnt).  Adjust corresponding insn_aux_data by copying
21414 * [0, off) and [off, end) to new locations, so the patched range stays zero
21415 */
21416static void adjust_insn_aux_data(struct bpf_verifier_env *env,
21417				 struct bpf_prog *new_prog, u32 off, u32 cnt)
21418{
21419	struct bpf_insn_aux_data *data = env->insn_aux_data;
21420	struct bpf_insn *insn = new_prog->insnsi;
21421	u32 old_seen = data[off].seen;
21422	u32 prog_len;
21423	int i;
21424
21425	/* aux info at OFF always needs adjustment, no matter fast path
21426	 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the
21427	 * original insn at old prog.
21428	 */
21429	data[off].zext_dst = insn_has_def32(insn + off + cnt - 1);
21430
21431	if (cnt == 1)
21432		return;
21433	prog_len = new_prog->len;
21434
21435	memmove(data + off + cnt - 1, data + off,
21436		sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
21437	memset(data + off, 0, sizeof(struct bpf_insn_aux_data) * (cnt - 1));
21438	for (i = off; i < off + cnt - 1; i++) {
21439		/* Expand insni[off]'s seen count to the patched range. */
21440		data[i].seen = old_seen;
21441		data[i].zext_dst = insn_has_def32(insn + i);
21442	}
21443}
21444
21445static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
21446{
21447	int i;
21448
21449	if (len == 1)
21450		return;
21451	/* NOTE: fake 'exit' subprog should be updated as well. */
21452	for (i = 0; i <= env->subprog_cnt; i++) {
21453		if (env->subprog_info[i].start <= off)
21454			continue;
21455		env->subprog_info[i].start += len - 1;
21456	}
21457}
21458
21459static void release_insn_arrays(struct bpf_verifier_env *env)
21460{
21461	int i;
21462
21463	for (i = 0; i < env->insn_array_map_cnt; i++)
21464		bpf_insn_array_release(env->insn_array_maps[i]);
21465}
21466
21467static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len)
21468{
21469	int i;
21470
21471	if (len == 1)
21472		return;
21473
21474	for (i = 0; i < env->insn_array_map_cnt; i++)
21475		bpf_insn_array_adjust(env->insn_array_maps[i], off, len);
21476}
21477
21478static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len)
21479{
21480	int i;
21481
21482	for (i = 0; i < env->insn_array_map_cnt; i++)
21483		bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len);
21484}
21485
21486static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len)
21487{
21488	struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;
21489	int i, sz = prog->aux->size_poke_tab;
21490	struct bpf_jit_poke_descriptor *desc;
21491
21492	for (i = 0; i < sz; i++) {
21493		desc = &tab[i];
21494		if (desc->insn_idx <= off)
21495			continue;
21496		desc->insn_idx += len - 1;
21497	}
21498}
21499
21500static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
21501					    const struct bpf_insn *patch, u32 len)
21502{
21503	struct bpf_prog *new_prog;
21504	struct bpf_insn_aux_data *new_data = NULL;
21505
21506	if (len > 1) {
21507		new_data = vrealloc(env->insn_aux_data,
21508				    array_size(env->prog->len + len - 1,
21509					       sizeof(struct bpf_insn_aux_data)),
21510				    GFP_KERNEL_ACCOUNT | __GFP_ZERO);
21511		if (!new_data)
21512			return NULL;
21513
21514		env->insn_aux_data = new_data;
21515	}
21516
21517	new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
21518	if (IS_ERR(new_prog)) {
21519		if (PTR_ERR(new_prog) == -ERANGE)
21520			verbose(env,
21521				"insn %d cannot be patched due to 16-bit range\n",
21522				env->insn_aux_data[off].orig_idx);
21523		return NULL;
21524	}
21525	adjust_insn_aux_data(env, new_prog, off, len);
21526	adjust_subprog_starts(env, off, len);
21527	adjust_insn_arrays(env, off, len);
21528	adjust_poke_descs(new_prog, off, len);
21529	return new_prog;
21530}
21531
21532/*
21533 * For all jmp insns in a given 'prog' that point to 'tgt_idx' insn adjust the
21534 * jump offset by 'delta'.
21535 */
21536static int adjust_jmp_off(struct bpf_prog *prog, u32 tgt_idx, u32 delta)
21537{
21538	struct bpf_insn *insn = prog->insnsi;
21539	u32 insn_cnt = prog->len, i;
21540	s32 imm;
21541	s16 off;
21542
21543	for (i = 0; i < insn_cnt; i++, insn++) {
21544		u8 code = insn->code;
21545
21546		if (tgt_idx <= i && i < tgt_idx + delta)
21547			continue;
21548
21549		if ((BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) ||
21550		    BPF_OP(code) == BPF_CALL || BPF_OP(code) == BPF_EXIT)
21551			continue;
21552
21553		if (insn->code == (BPF_JMP32 | BPF_JA)) {
21554			if (i + 1 + insn->imm != tgt_idx)
21555				continue;
21556			if (check_add_overflow(insn->imm, delta, &imm))
21557				return -ERANGE;
21558			insn->imm = imm;
21559		} else {
21560			if (i + 1 + insn->off != tgt_idx)
21561				continue;
21562			if (check_add_overflow(insn->off, delta, &off))
21563				return -ERANGE;
21564			insn->off = off;
21565		}
21566	}
21567	return 0;
21568}
21569
21570static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
21571					      u32 off, u32 cnt)
21572{
21573	int i, j;
21574
21575	/* find first prog starting at or after off (first to remove) */
21576	for (i = 0; i < env->subprog_cnt; i++)
21577		if (env->subprog_info[i].start >= off)
21578			break;
21579	/* find first prog starting at or after off + cnt (first to stay) */
21580	for (j = i; j < env->subprog_cnt; j++)
21581		if (env->subprog_info[j].start >= off + cnt)
21582			break;
21583	/* if j doesn't start exactly at off + cnt, we are just removing
21584	 * the front of previous prog
21585	 */
21586	if (env->subprog_info[j].start != off + cnt)
21587		j--;
21588
21589	if (j > i) {
21590		struct bpf_prog_aux *aux = env->prog->aux;
21591		int move;
21592
21593		/* move fake 'exit' subprog as well */
21594		move = env->subprog_cnt + 1 - j;
21595
21596		memmove(env->subprog_info + i,
21597			env->subprog_info + j,
21598			sizeof(*env->subprog_info) * move);
21599		env->subprog_cnt -= j - i;
21600
21601		/* remove func_info */
21602		if (aux->func_info) {
21603			move = aux->func_info_cnt - j;
21604
21605			memmove(aux->func_info + i,
21606				aux->func_info + j,
21607				sizeof(*aux->func_info) * move);
21608			aux->func_info_cnt -= j - i;
21609			/* func_info->insn_off is set after all code rewrites,
21610			 * in adjust_btf_func() - no need to adjust
21611			 */
21612		}
21613	} else {
21614		/* convert i from "first prog to remove" to "first to adjust" */
21615		if (env->subprog_info[i].start == off)
21616			i++;
21617	}
21618
21619	/* update fake 'exit' subprog as well */
21620	for (; i <= env->subprog_cnt; i++)
21621		env->subprog_info[i].start -= cnt;
21622
21623	return 0;
21624}
21625
21626static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,
21627				      u32 cnt)
21628{
21629	struct bpf_prog *prog = env->prog;
21630	u32 i, l_off, l_cnt, nr_linfo;
21631	struct bpf_line_info *linfo;
21632
21633	nr_linfo = prog->aux->nr_linfo;
21634	if (!nr_linfo)
21635		return 0;
21636
21637	linfo = prog->aux->linfo;
21638
21639	/* find first line info to remove, count lines to be removed */
21640	for (i = 0; i < nr_linfo; i++)
21641		if (linfo[i].insn_off >= off)
21642			break;
21643
21644	l_off = i;
21645	l_cnt = 0;
21646	for (; i < nr_linfo; i++)
21647		if (linfo[i].insn_off < off + cnt)
21648			l_cnt++;
21649		else
21650			break;
21651
21652	/* First live insn doesn't match first live linfo, it needs to "inherit"
21653	 * last removed linfo.  prog is already modified, so prog->len == off
21654	 * means no live instructions after (tail of the program was removed).
21655	 */
21656	if (prog->len != off && l_cnt &&
21657	    (i == nr_linfo || linfo[i].insn_off != off + cnt)) {
21658		l_cnt--;
21659		linfo[--i].insn_off = off + cnt;
21660	}
21661
21662	/* remove the line info which refer to the removed instructions */
21663	if (l_cnt) {
21664		memmove(linfo + l_off, linfo + i,
21665			sizeof(*linfo) * (nr_linfo - i));
21666
21667		prog->aux->nr_linfo -= l_cnt;
21668		nr_linfo = prog->aux->nr_linfo;
21669	}
21670
21671	/* pull all linfo[i].insn_off >= off + cnt in by cnt */
21672	for (i = l_off; i < nr_linfo; i++)
21673		linfo[i].insn_off -= cnt;
21674
21675	/* fix up all subprogs (incl. 'exit') which start >= off */
21676	for (i = 0; i <= env->subprog_cnt; i++)
21677		if (env->subprog_info[i].linfo_idx > l_off) {
21678			/* program may have started in the removed region but
21679			 * may not be fully removed
21680			 */
21681			if (env->subprog_info[i].linfo_idx >= l_off + l_cnt)
21682				env->subprog_info[i].linfo_idx -= l_cnt;
21683			else
21684				env->subprog_info[i].linfo_idx = l_off;
21685		}
21686
21687	return 0;
21688}
21689
21690/*
21691 * Clean up dynamically allocated fields of aux data for instructions [start, ...]
21692 */
21693static void clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len)
21694{
21695	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
21696	struct bpf_insn *insns = env->prog->insnsi;
21697	int end = start + len;
21698	int i;
21699
21700	for (i = start; i < end; i++) {
21701		if (aux_data[i].jt) {
21702			kvfree(aux_data[i].jt);
21703			aux_data[i].jt = NULL;
21704		}
21705
21706		if (bpf_is_ldimm64(&insns[i]))
21707			i++;
21708	}
21709}
21710
21711static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
21712{
21713	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
21714	unsigned int orig_prog_len = env->prog->len;
21715	int err;
21716
21717	if (bpf_prog_is_offloaded(env->prog->aux))
21718		bpf_prog_offload_remove_insns(env, off, cnt);
21719
21720	/* Should be called before bpf_remove_insns, as it uses prog->insnsi */
21721	clear_insn_aux_data(env, off, cnt);
21722
21723	err = bpf_remove_insns(env->prog, off, cnt);
21724	if (err)
21725		return err;
21726
21727	err = adjust_subprog_starts_after_remove(env, off, cnt);
21728	if (err)
21729		return err;
21730
21731	err = bpf_adj_linfo_after_remove(env, off, cnt);
21732	if (err)
21733		return err;
21734
21735	adjust_insn_arrays_after_remove(env, off, cnt);
21736
21737	memmove(aux_data + off,	aux_data + off + cnt,
21738		sizeof(*aux_data) * (orig_prog_len - off - cnt));
21739
21740	return 0;
21741}
21742
21743/* The verifier does more data flow analysis than llvm and will not
21744 * explore branches that are dead at run time. Malicious programs can
21745 * have dead code too. Therefore replace all dead at-run-time code
21746 * with 'ja -1'.
21747 *
21748 * Just nops are not optimal, e.g. if they would sit at the end of the
21749 * program and through another bug we would manage to jump there, then
21750 * we'd execute beyond program memory otherwise. Returning exception
21751 * code also wouldn't work since we can have subprogs where the dead
21752 * code could be located.
21753 */
21754static void sanitize_dead_code(struct bpf_verifier_env *env)
21755{
21756	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
21757	struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1);
21758	struct bpf_insn *insn = env->prog->insnsi;
21759	const int insn_cnt = env->prog->len;
21760	int i;
21761
21762	for (i = 0; i < insn_cnt; i++) {
21763		if (aux_data[i].seen)
21764			continue;
21765		memcpy(insn + i, &trap, sizeof(trap));
21766		aux_data[i].zext_dst = false;
21767	}
21768}
21769
21770static bool insn_is_cond_jump(u8 code)
21771{
21772	u8 op;
21773
21774	op = BPF_OP(code);
21775	if (BPF_CLASS(code) == BPF_JMP32)
21776		return op != BPF_JA;
21777
21778	if (BPF_CLASS(code) != BPF_JMP)
21779		return false;
21780
21781	return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL;
21782}
21783
21784static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
21785{
21786	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
21787	struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
21788	struct bpf_insn *insn = env->prog->insnsi;
21789	const int insn_cnt = env->prog->len;
21790	int i;
21791
21792	for (i = 0; i < insn_cnt; i++, insn++) {
21793		if (!insn_is_cond_jump(insn->code))
21794			continue;
21795
21796		if (!aux_data[i + 1].seen)
21797			ja.off = insn->off;
21798		else if (!aux_data[i + 1 + insn->off].seen)
21799			ja.off = 0;
21800		else
21801			continue;
21802
21803		if (bpf_prog_is_offloaded(env->prog->aux))
21804			bpf_prog_offload_replace_insn(env, i, &ja);
21805
21806		memcpy(insn, &ja, sizeof(ja));
21807	}
21808}
21809
21810static int opt_remove_dead_code(struct bpf_verifier_env *env)
21811{
21812	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
21813	int insn_cnt = env->prog->len;
21814	int i, err;
21815
21816	for (i = 0; i < insn_cnt; i++) {
21817		int j;
21818
21819		j = 0;
21820		while (i + j < insn_cnt && !aux_data[i + j].seen)
21821			j++;
21822		if (!j)
21823			continue;
21824
21825		err = verifier_remove_insns(env, i, j);
21826		if (err)
21827			return err;
21828		insn_cnt = env->prog->len;
21829	}
21830
21831	return 0;
21832}
21833
21834static const struct bpf_insn NOP = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
21835static const struct bpf_insn MAY_GOTO_0 = BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 0, 0);
21836
21837static int opt_remove_nops(struct bpf_verifier_env *env)
21838{
21839	struct bpf_insn *insn = env->prog->insnsi;
21840	int insn_cnt = env->prog->len;
21841	bool is_may_goto_0, is_ja;
21842	int i, err;
21843
21844	for (i = 0; i < insn_cnt; i++) {
21845		is_may_goto_0 = !memcmp(&insn[i], &MAY_GOTO_0, sizeof(MAY_GOTO_0));
21846		is_ja = !memcmp(&insn[i], &NOP, sizeof(NOP));
21847
21848		if (!is_may_goto_0 && !is_ja)
21849			continue;
21850
21851		err = verifier_remove_insns(env, i, 1);
21852		if (err)
21853			return err;
21854		insn_cnt--;
21855		/* Go back one insn to catch may_goto +1; may_goto +0 sequence */
21856		i -= (is_may_goto_0 && i > 0) ? 2 : 1;
21857	}
21858
21859	return 0;
21860}
21861
21862static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,
21863					 const union bpf_attr *attr)
21864{
21865	struct bpf_insn *patch;
21866	/* use env->insn_buf as two independent buffers */
21867	struct bpf_insn *zext_patch = env->insn_buf;
21868	struct bpf_insn *rnd_hi32_patch = &env->insn_buf[2];
21869	struct bpf_insn_aux_data *aux = env->insn_aux_data;
21870	int i, patch_len, delta = 0, len = env->prog->len;
21871	struct bpf_insn *insns = env->prog->insnsi;
21872	struct bpf_prog *new_prog;
21873	bool rnd_hi32;
21874
21875	rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32;
21876	zext_patch[1] = BPF_ZEXT_REG(0);
21877	rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0);
21878	rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32);
21879	rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX);
21880	for (i = 0; i < len; i++) {
21881		int adj_idx = i + delta;
21882		struct bpf_insn insn;
21883		int load_reg;
21884
21885		insn = insns[adj_idx];
21886		load_reg = insn_def_regno(&insn);
21887		if (!aux[adj_idx].zext_dst) {
21888			u8 code, class;
21889			u32 imm_rnd;
21890
21891			if (!rnd_hi32)
21892				continue;
21893
21894			code = insn.code;
21895			class = BPF_CLASS(code);
21896			if (load_reg == -1)
21897				continue;
21898
21899			/* NOTE: arg "reg" (the fourth one) is only used for
21900			 *       BPF_STX + SRC_OP, so it is safe to pass NULL
21901			 *       here.
21902			 */
21903			if (is_reg64(&insn, load_reg, NULL, DST_OP)) {
21904				if (class == BPF_LD &&
21905				    BPF_MODE(code) == BPF_IMM)
21906					i++;
21907				continue;
21908			}
21909
21910			/* ctx load could be transformed into wider load. */
21911			if (class == BPF_LDX &&
21912			    aux[adj_idx].ptr_type == PTR_TO_CTX)
21913				continue;
21914
21915			imm_rnd = get_random_u32();
21916			rnd_hi32_patch[0] = insn;
21917			rnd_hi32_patch[1].imm = imm_rnd;
21918			rnd_hi32_patch[3].dst_reg = load_reg;
21919			patch = rnd_hi32_patch;
21920			patch_len = 4;
21921			goto apply_patch_buffer;
21922		}
21923
21924		/* Add in an zero-extend instruction if a) the JIT has requested
21925		 * it or b) it's a CMPXCHG.
21926		 *
21927		 * The latter is because: BPF_CMPXCHG always loads a value into
21928		 * R0, therefore always zero-extends. However some archs'
21929		 * equivalent instruction only does this load when the
21930		 * comparison is successful. This detail of CMPXCHG is
21931		 * orthogonal to the general zero-extension behaviour of the
21932		 * CPU, so it's treated independently of bpf_jit_needs_zext.
21933		 */
21934		if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn))
21935			continue;
21936
21937		/* Zero-extension is done by the caller. */
21938		if (bpf_pseudo_kfunc_call(&insn))
21939			continue;
21940
21941		if (verifier_bug_if(load_reg == -1, env,
21942				    "zext_dst is set, but no reg is defined"))
21943			return -EFAULT;
21944
21945		zext_patch[0] = insn;
21946		zext_patch[1].dst_reg = load_reg;
21947		zext_patch[1].src_reg = load_reg;
21948		patch = zext_patch;
21949		patch_len = 2;
21950apply_patch_buffer:
21951		new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len);
21952		if (!new_prog)
21953			return -ENOMEM;
21954		env->prog = new_prog;
21955		insns = new_prog->insnsi;
21956		aux = env->insn_aux_data;
21957		delta += patch_len - 1;
21958	}
21959
21960	return 0;
21961}
21962
21963/* convert load instructions that access fields of a context type into a
21964 * sequence of instructions that access fields of the underlying structure:
21965 *     struct __sk_buff    -> struct sk_buff
21966 *     struct bpf_sock_ops -> struct sock
21967 */
21968static int convert_ctx_accesses(struct bpf_verifier_env *env)
21969{
21970	struct bpf_subprog_info *subprogs = env->subprog_info;
21971	const struct bpf_verifier_ops *ops = env->ops;
21972	int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0;
21973	const int insn_cnt = env->prog->len;
21974	struct bpf_insn *epilogue_buf = env->epilogue_buf;
21975	struct bpf_insn *insn_buf = env->insn_buf;
21976	struct bpf_insn *insn;
21977	u32 target_size, size_default, off;
21978	struct bpf_prog *new_prog;
21979	enum bpf_access_type type;
21980	bool is_narrower_load;
21981	int epilogue_idx = 0;
21982
21983	if (ops->gen_epilogue) {
21984		epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog,
21985						 -(subprogs[0].stack_depth + 8));
21986		if (epilogue_cnt >= INSN_BUF_SIZE) {
21987			verifier_bug(env, "epilogue is too long");
21988			return -EFAULT;
21989		} else if (epilogue_cnt) {
21990			/* Save the ARG_PTR_TO_CTX for the epilogue to use */
21991			cnt = 0;
21992			subprogs[0].stack_depth += 8;
21993			insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1,
21994						      -subprogs[0].stack_depth);
21995			insn_buf[cnt++] = env->prog->insnsi[0];
21996			new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
21997			if (!new_prog)
21998				return -ENOMEM;
21999			env->prog = new_prog;
22000			delta += cnt - 1;
22001
22002			ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1);
22003			if (ret < 0)
22004				return ret;
22005		}
22006	}
22007
22008	if (ops->gen_prologue || env->seen_direct_write) {
22009		if (!ops->gen_prologue) {
22010			verifier_bug(env, "gen_prologue is null");
22011			return -EFAULT;
22012		}
22013		cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,
22014					env->prog);
22015		if (cnt >= INSN_BUF_SIZE) {
22016			verifier_bug(env, "prologue is too long");
22017			return -EFAULT;
22018		} else if (cnt) {
22019			new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
22020			if (!new_prog)
22021				return -ENOMEM;
22022
22023			env->prog = new_prog;
22024			delta += cnt - 1;
22025
22026			ret = add_kfunc_in_insns(env, insn_buf, cnt - 1);
22027			if (ret < 0)
22028				return ret;
22029		}
22030	}
22031
22032	if (delta)
22033		WARN_ON(adjust_jmp_off(env->prog, 0, delta));
22034
22035	if (bpf_prog_is_offloaded(env->prog->aux))
22036		return 0;
22037
22038	insn = env->prog->insnsi + delta;
22039
22040	for (i = 0; i < insn_cnt; i++, insn++) {
22041		bpf_convert_ctx_access_t convert_ctx_access;
22042		u8 mode;
22043
22044		if (env->insn_aux_data[i + delta].nospec) {
22045			WARN_ON_ONCE(env->insn_aux_data[i + delta].alu_state);
22046			struct bpf_insn *patch = insn_buf;
22047
22048			*patch++ = BPF_ST_NOSPEC();
22049			*patch++ = *insn;
22050			cnt = patch - insn_buf;
22051			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
22052			if (!new_prog)
22053				return -ENOMEM;
22054
22055			delta    += cnt - 1;
22056			env->prog = new_prog;
22057			insn      = new_prog->insnsi + i + delta;
22058			/* This can not be easily merged with the
22059			 * nospec_result-case, because an insn may require a
22060			 * nospec before and after itself. Therefore also do not
22061			 * 'continue' here but potentially apply further
22062			 * patching to insn. *insn should equal patch[1] now.
22063			 */
22064		}
22065
22066		if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) ||
22067		    insn->code == (BPF_LDX | BPF_MEM | BPF_H) ||
22068		    insn->code == (BPF_LDX | BPF_MEM | BPF_W) ||
22069		    insn->code == (BPF_LDX | BPF_MEM | BPF_DW) ||
22070		    insn->code == (BPF_LDX | BPF_MEMSX | BPF_B) ||
22071		    insn->code == (BPF_LDX | BPF_MEMSX | BPF_H) ||
22072		    insn->code == (BPF_LDX | BPF_MEMSX | BPF_W)) {
22073			type = BPF_READ;
22074		} else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) ||
22075			   insn->code == (BPF_STX | BPF_MEM | BPF_H) ||
22076			   insn->code == (BPF_STX | BPF_MEM | BPF_W) ||
22077			   insn->code == (BPF_STX | BPF_MEM | BPF_DW) ||
22078			   insn->code == (BPF_ST | BPF_MEM | BPF_B) ||
22079			   insn->code == (BPF_ST | BPF_MEM | BPF_H) ||
22080			   insn->code == (BPF_ST | BPF_MEM | BPF_W) ||
22081			   insn->code == (BPF_ST | BPF_MEM | BPF_DW)) {
22082			type = BPF_WRITE;
22083		} else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_B) ||
22084			    insn->code == (BPF_STX | BPF_ATOMIC | BPF_H) ||
22085			    insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) ||
22086			    insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) &&
22087			   env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) {
22088			insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code);
22089			env->prog->aux->num_exentries++;
22090			continue;
22091		} else if (insn->code == (BPF_JMP | BPF_EXIT) &&
22092			   epilogue_cnt &&
22093			   i + delta < subprogs[1].start) {
22094			/* Generate epilogue for the main prog */
22095			if (epilogue_idx) {
22096				/* jump back to the earlier generated epilogue */
22097				insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - delta - 1);
22098				cnt = 1;
22099			} else {
22100				memcpy(insn_buf, epilogue_buf,
22101				       epilogue_cnt * sizeof(*epilogue_buf));
22102				cnt = epilogue_cnt;
22103				/* epilogue_idx cannot be 0. It must have at
22104				 * least one ctx ptr saving insn before the
22105				 * epilogue.
22106				 */
22107				epilogue_idx = i + delta;
22108			}
22109			goto patch_insn_buf;
22110		} else {
22111			continue;
22112		}
22113
22114		if (type == BPF_WRITE &&
22115		    env->insn_aux_data[i + delta].nospec_result) {
22116			/* nospec_result is only used to mitigate Spectre v4 and
22117			 * to limit verification-time for Spectre v1.
22118			 */
22119			struct bpf_insn *patch = insn_buf;
22120
22121			*patch++ = *insn;
22122			*patch++ = BPF_ST_NOSPEC();
22123			cnt = patch - insn_buf;
22124			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
22125			if (!new_prog)
22126				return -ENOMEM;
22127
22128			delta    += cnt - 1;
22129			env->prog = new_prog;
22130			insn      = new_prog->insnsi + i + delta;
22131			continue;
22132		}
22133
22134		switch ((int)env->insn_aux_data[i + delta].ptr_type) {
22135		case PTR_TO_CTX:
22136			if (!ops->convert_ctx_access)
22137				continue;
22138			convert_ctx_access = ops->convert_ctx_access;
22139			break;
22140		case PTR_TO_SOCKET:
22141		case PTR_TO_SOCK_COMMON:
22142			convert_ctx_access = bpf_sock_convert_ctx_access;
22143			break;
22144		case PTR_TO_TCP_SOCK:
22145			convert_ctx_access = bpf_tcp_sock_convert_ctx_access;
22146			break;
22147		case PTR_TO_XDP_SOCK:
22148			convert_ctx_access = bpf_xdp_sock_convert_ctx_access;
22149			break;
22150		case PTR_TO_BTF_ID:
22151		case PTR_TO_BTF_ID | PTR_UNTRUSTED:
22152		/* PTR_TO_BTF_ID | MEM_ALLOC always has a valid lifetime, unlike
22153		 * PTR_TO_BTF_ID, and an active ref_obj_id, but the same cannot
22154		 * be said once it is marked PTR_UNTRUSTED, hence we must handle
22155		 * any faults for loads into such types. BPF_WRITE is disallowed
22156		 * for this case.
22157		 */
22158		case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED:
22159		case PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED:
22160			if (type == BPF_READ) {
22161				if (BPF_MODE(insn->code) == BPF_MEM)
22162					insn->code = BPF_LDX | BPF_PROBE_MEM |
22163						     BPF_SIZE((insn)->code);
22164				else
22165					insn->code = BPF_LDX | BPF_PROBE_MEMSX |
22166						     BPF_SIZE((insn)->code);
22167				env->prog->aux->num_exentries++;
22168			}
22169			continue;
22170		case PTR_TO_ARENA:
22171			if (BPF_MODE(insn->code) == BPF_MEMSX) {
22172				if (!bpf_jit_supports_insn(insn, true)) {
22173					verbose(env, "sign extending loads from arena are not supported yet\n");
22174					return -EOPNOTSUPP;
22175				}
22176				insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32SX | BPF_SIZE(insn->code);
22177			} else {
22178				insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code);
22179			}
22180			env->prog->aux->num_exentries++;
22181			continue;
22182		default:
22183			continue;
22184		}
22185
22186		ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;
22187		size = BPF_LDST_BYTES(insn);
22188		mode = BPF_MODE(insn->code);
22189
22190		/* If the read access is a narrower load of the field,
22191		 * convert to a 4/8-byte load, to minimum program type specific
22192		 * convert_ctx_access changes. If conversion is successful,
22193		 * we will apply proper mask to the result.
22194		 */
22195		is_narrower_load = size < ctx_field_size;
22196		size_default = bpf_ctx_off_adjust_machine(ctx_field_size);
22197		off = insn->off;
22198		if (is_narrower_load) {
22199			u8 size_code;
22200
22201			if (type == BPF_WRITE) {
22202				verifier_bug(env, "narrow ctx access misconfigured");
22203				return -EFAULT;
22204			}
22205
22206			size_code = BPF_H;
22207			if (ctx_field_size == 4)
22208				size_code = BPF_W;
22209			else if (ctx_field_size == 8)
22210				size_code = BPF_DW;
22211
22212			insn->off = off & ~(size_default - 1);
22213			insn->code = BPF_LDX | BPF_MEM | size_code;
22214		}
22215
22216		target_size = 0;
22217		cnt = convert_ctx_access(type, insn, insn_buf, env->prog,
22218					 &target_size);
22219		if (cnt == 0 || cnt >= INSN_BUF_SIZE ||
22220		    (ctx_field_size && !target_size)) {
22221			verifier_bug(env, "error during ctx access conversion (%d)", cnt);
22222			return -EFAULT;
22223		}
22224
22225		if (is_narrower_load && size < target_size) {
22226			u8 shift = bpf_ctx_narrow_access_offset(
22227				off, size, size_default) * 8;
22228			if (shift && cnt + 1 >= INSN_BUF_SIZE) {
22229				verifier_bug(env, "narrow ctx load misconfigured");
22230				return -EFAULT;
22231			}
22232			if (ctx_field_size <= 4) {
22233				if (shift)
22234					insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH,
22235									insn->dst_reg,
22236									shift);
22237				insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,
22238								(1 << size * 8) - 1);
22239			} else {
22240				if (shift)
22241					insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH,
22242									insn->dst_reg,
22243									shift);
22244				insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,
22245								(1ULL << size * 8) - 1);
22246			}
22247		}
22248		if (mode == BPF_MEMSX)
22249			insn_buf[cnt++] = BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X,
22250						       insn->dst_reg, insn->dst_reg,
22251						       size * 8, 0);
22252
22253patch_insn_buf:
22254		new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
22255		if (!new_prog)
22256			return -ENOMEM;
22257
22258		delta += cnt - 1;
22259
22260		/* keep walking new program and skip insns we just inserted */
22261		env->prog = new_prog;
22262		insn      = new_prog->insnsi + i + delta;
22263	}
22264
22265	return 0;
22266}
22267
22268static int jit_subprogs(struct bpf_verifier_env *env)
22269{
22270	struct bpf_prog *prog = env->prog, **func, *tmp;
22271	int i, j, subprog_start, subprog_end = 0, len, subprog;
22272	struct bpf_map *map_ptr;
22273	struct bpf_insn *insn;
22274	void *old_bpf_func;
22275	int err, num_exentries;
22276	int old_len, subprog_start_adjustment = 0;
22277
22278	if (env->subprog_cnt <= 1)
22279		return 0;
22280
22281	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
22282		if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn))
22283			continue;
22284
22285		/* Upon error here we cannot fall back to interpreter but
22286		 * need a hard reject of the program. Thus -EFAULT is
22287		 * propagated in any case.
22288		 */
22289		subprog = find_subprog(env, i + insn->imm + 1);
22290		if (verifier_bug_if(subprog < 0, env, "No program to jit at insn %d",
22291				    i + insn->imm + 1))
22292			return -EFAULT;
22293		/* temporarily remember subprog id inside insn instead of
22294		 * aux_data, since next loop will split up all insns into funcs
22295		 */
22296		insn->off = subprog;
22297		/* remember original imm in case JIT fails and fallback
22298		 * to interpreter will be needed
22299		 */
22300		env->insn_aux_data[i].call_imm = insn->imm;
22301		/* point imm to __bpf_call_base+1 from JITs point of view */
22302		insn->imm = 1;
22303		if (bpf_pseudo_func(insn)) {
22304#if defined(MODULES_VADDR)
22305			u64 addr = MODULES_VADDR;
22306#else
22307			u64 addr = VMALLOC_START;
22308#endif
22309			/* jit (e.g. x86_64) may emit fewer instructions
22310			 * if it learns a u32 imm is the same as a u64 imm.
22311			 * Set close enough to possible prog address.
22312			 */
22313			insn[0].imm = (u32)addr;
22314			insn[1].imm = addr >> 32;
22315		}
22316	}
22317
22318	err = bpf_prog_alloc_jited_linfo(prog);
22319	if (err)
22320		goto out_undo_insn;
22321
22322	err = -ENOMEM;
22323	func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);
22324	if (!func)
22325		goto out_undo_insn;
22326
22327	for (i = 0; i < env->subprog_cnt; i++) {
22328		subprog_start = subprog_end;
22329		subprog_end = env->subprog_info[i + 1].start;
22330
22331		len = subprog_end - subprog_start;
22332		/* bpf_prog_run() doesn't call subprogs directly,
22333		 * hence main prog stats include the runtime of subprogs.
22334		 * subprogs don't have IDs and not reachable via prog_get_next_id
22335		 * func[i]->stats will never be accessed and stays NULL
22336		 */
22337		func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
22338		if (!func[i])
22339			goto out_free;
22340		memcpy(func[i]->insnsi, &prog->insnsi[subprog_start],
22341		       len * sizeof(struct bpf_insn));
22342		func[i]->type = prog->type;
22343		func[i]->len = len;
22344		if (bpf_prog_calc_tag(func[i]))
22345			goto out_free;
22346		func[i]->is_func = 1;
22347		func[i]->sleepable = prog->sleepable;
22348		func[i]->aux->func_idx = i;
22349		/* Below members will be freed only at prog->aux */
22350		func[i]->aux->btf = prog->aux->btf;
22351		func[i]->aux->subprog_start = subprog_start + subprog_start_adjustment;
22352		func[i]->aux->func_info = prog->aux->func_info;
22353		func[i]->aux->func_info_cnt = prog->aux->func_info_cnt;
22354		func[i]->aux->poke_tab = prog->aux->poke_tab;
22355		func[i]->aux->size_poke_tab = prog->aux->size_poke_tab;
22356		func[i]->aux->main_prog_aux = prog->aux;
22357
22358		for (j = 0; j < prog->aux->size_poke_tab; j++) {
22359			struct bpf_jit_poke_descriptor *poke;
22360
22361			poke = &prog->aux->poke_tab[j];
22362			if (poke->insn_idx < subprog_end &&
22363			    poke->insn_idx >= subprog_start)
22364				poke->aux = func[i]->aux;
22365		}
22366
22367		func[i]->aux->name[0] = 'F';
22368		func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
22369		if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE)
22370			func[i]->aux->jits_use_priv_stack = true;
22371
22372		func[i]->jit_requested = 1;
22373		func[i]->blinding_requested = prog->blinding_requested;
22374		func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;
22375		func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab;
22376		func[i]->aux->linfo = prog->aux->linfo;
22377		func[i]->aux->nr_linfo = prog->aux->nr_linfo;
22378		func[i]->aux->jited_linfo = prog->aux->jited_linfo;
22379		func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
22380		func[i]->aux->arena = prog->aux->arena;
22381		func[i]->aux->used_maps = env->used_maps;
22382		func[i]->aux->used_map_cnt = env->used_map_cnt;
22383		num_exentries = 0;
22384		insn = func[i]->insnsi;
22385		for (j = 0; j < func[i]->len; j++, insn++) {
22386			if (BPF_CLASS(insn->code) == BPF_LDX &&
22387			    (BPF_MODE(insn->code) == BPF_PROBE_MEM ||
22388			     BPF_MODE(insn->code) == BPF_PROBE_MEM32 ||
22389			     BPF_MODE(insn->code) == BPF_PROBE_MEM32SX ||
22390			     BPF_MODE(insn->code) == BPF_PROBE_MEMSX))
22391				num_exentries++;
22392			if ((BPF_CLASS(insn->code) == BPF_STX ||
22393			     BPF_CLASS(insn->code) == BPF_ST) &&
22394			     BPF_MODE(insn->code) == BPF_PROBE_MEM32)
22395				num_exentries++;
22396			if (BPF_CLASS(insn->code) == BPF_STX &&
22397			     BPF_MODE(insn->code) == BPF_PROBE_ATOMIC)
22398				num_exentries++;
22399		}
22400		func[i]->aux->num_exentries = num_exentries;
22401		func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;
22402		func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;
22403		func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;
22404		func[i]->aux->might_sleep = env->subprog_info[i].might_sleep;
22405		if (!i)
22406			func[i]->aux->exception_boundary = env->seen_exception;
22407
22408		/*
22409		 * To properly pass the absolute subprog start to jit
22410		 * all instruction adjustments should be accumulated
22411		 */
22412		old_len = func[i]->len;
22413		func[i] = bpf_int_jit_compile(func[i]);
22414		subprog_start_adjustment += func[i]->len - old_len;
22415
22416		if (!func[i]->jited) {
22417			err = -ENOTSUPP;
22418			goto out_free;
22419		}
22420		cond_resched();
22421	}
22422
22423	/* at this point all bpf functions were successfully JITed
22424	 * now populate all bpf_calls with correct addresses and
22425	 * run last pass of JIT
22426	 */
22427	for (i = 0; i < env->subprog_cnt; i++) {
22428		insn = func[i]->insnsi;
22429		for (j = 0; j < func[i]->len; j++, insn++) {
22430			if (bpf_pseudo_func(insn)) {
22431				subprog = insn->off;
22432				insn[0].imm = (u32)(long)func[subprog]->bpf_func;
22433				insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32;
22434				continue;
22435			}
22436			if (!bpf_pseudo_call(insn))
22437				continue;
22438			subprog = insn->off;
22439			insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func);
22440		}
22441
22442		/* we use the aux data to keep a list of the start addresses
22443		 * of the JITed images for each function in the program
22444		 *
22445		 * for some architectures, such as powerpc64, the imm field
22446		 * might not be large enough to hold the offset of the start
22447		 * address of the callee's JITed image from __bpf_call_base
22448		 *
22449		 * in such cases, we can lookup the start address of a callee
22450		 * by using its subprog id, available from the off field of
22451		 * the call instruction, as an index for this list
22452		 */
22453		func[i]->aux->func = func;
22454		func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;
22455		func[i]->aux->real_func_cnt = env->subprog_cnt;
22456	}
22457	for (i = 0; i < env->subprog_cnt; i++) {
22458		old_bpf_func = func[i]->bpf_func;
22459		tmp = bpf_int_jit_compile(func[i]);
22460		if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
22461			verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
22462			err = -ENOTSUPP;
22463			goto out_free;
22464		}
22465		cond_resched();
22466	}
22467
22468	/*
22469	 * Cleanup func[i]->aux fields which aren't required
22470	 * or can become invalid in future
22471	 */
22472	for (i = 0; i < env->subprog_cnt; i++) {
22473		func[i]->aux->used_maps = NULL;
22474		func[i]->aux->used_map_cnt = 0;
22475	}
22476
22477	/* finally lock prog and jit images for all functions and
22478	 * populate kallsysm. Begin at the first subprogram, since
22479	 * bpf_prog_load will add the kallsyms for the main program.
22480	 */
22481	for (i = 1; i < env->subprog_cnt; i++) {
22482		err = bpf_prog_lock_ro(func[i]);
22483		if (err)
22484			goto out_free;
22485	}
22486
22487	for (i = 1; i < env->subprog_cnt; i++)
22488		bpf_prog_kallsyms_add(func[i]);
22489
22490	/* Last step: make now unused interpreter insns from main
22491	 * prog consistent for later dump requests, so they can
22492	 * later look the same as if they were interpreted only.
22493	 */
22494	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
22495		if (bpf_pseudo_func(insn)) {
22496			insn[0].imm = env->insn_aux_data[i].call_imm;
22497			insn[1].imm = insn->off;
22498			insn->off = 0;
22499			continue;
22500		}
22501		if (!bpf_pseudo_call(insn))
22502			continue;
22503		insn->off = env->insn_aux_data[i].call_imm;
22504		subprog = find_subprog(env, i + insn->off + 1);
22505		insn->imm = subprog;
22506	}
22507
22508	prog->jited = 1;
22509	prog->bpf_func = func[0]->bpf_func;
22510	prog->jited_len = func[0]->jited_len;
22511	prog->aux->extable = func[0]->aux->extable;
22512	prog->aux->num_exentries = func[0]->aux->num_exentries;
22513	prog->aux->func = func;
22514	prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;
22515	prog->aux->real_func_cnt = env->subprog_cnt;
22516	prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func;
22517	prog->aux->exception_boundary = func[0]->aux->exception_boundary;
22518	bpf_prog_jit_attempt_done(prog);
22519	return 0;
22520out_free:
22521	/* We failed JIT'ing, so at this point we need to unregister poke
22522	 * descriptors from subprogs, so that kernel is not attempting to
22523	 * patch it anymore as we're freeing the subprog JIT memory.
22524	 */
22525	for (i = 0; i < prog->aux->size_poke_tab; i++) {
22526		map_ptr = prog->aux->poke_tab[i].tail_call.map;
22527		map_ptr->ops->map_poke_untrack(map_ptr, prog->aux);
22528	}
22529	/* At this point we're guaranteed that poke descriptors are not
22530	 * live anymore. We can just unlink its descriptor table as it's
22531	 * released with the main prog.
22532	 */
22533	for (i = 0; i < env->subprog_cnt; i++) {
22534		if (!func[i])
22535			continue;
22536		func[i]->aux->poke_tab = NULL;
22537		bpf_jit_free(func[i]);
22538	}
22539	kfree(func);
22540out_undo_insn:
22541	/* cleanup main prog to be interpreted */
22542	prog->jit_requested = 0;
22543	prog->blinding_requested = 0;
22544	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
22545		if (!bpf_pseudo_call(insn))
22546			continue;
22547		insn->off = 0;
22548		insn->imm = env->insn_aux_data[i].call_imm;
22549	}
22550	bpf_prog_jit_attempt_done(prog);
22551	return err;
22552}
22553
22554static int fixup_call_args(struct bpf_verifier_env *env)
22555{
22556#ifndef CONFIG_BPF_JIT_ALWAYS_ON
22557	struct bpf_prog *prog = env->prog;
22558	struct bpf_insn *insn = prog->insnsi;
22559	bool has_kfunc_call = bpf_prog_has_kfunc_call(prog);
22560	int i, depth;
22561#endif
22562	int err = 0;
22563
22564	if (env->prog->jit_requested &&
22565	    !bpf_prog_is_offloaded(env->prog->aux)) {
22566		err = jit_subprogs(env);
22567		if (err == 0)
22568			return 0;
22569		if (err == -EFAULT)
22570			return err;
22571	}
22572#ifndef CONFIG_BPF_JIT_ALWAYS_ON
22573	if (has_kfunc_call) {
22574		verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");
22575		return -EINVAL;
22576	}
22577	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
22578		/* When JIT fails the progs with bpf2bpf calls and tail_calls
22579		 * have to be rejected, since interpreter doesn't support them yet.
22580		 */
22581		verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
22582		return -EINVAL;
22583	}
22584	for (i = 0; i < prog->len; i++, insn++) {
22585		if (bpf_pseudo_func(insn)) {
22586			/* When JIT fails the progs with callback calls
22587			 * have to be rejected, since interpreter doesn't support them yet.
22588			 */
22589			verbose(env, "callbacks are not allowed in non-JITed programs\n");
22590			return -EINVAL;
22591		}
22592
22593		if (!bpf_pseudo_call(insn))
22594			continue;
22595		depth = get_callee_stack_depth(env, insn, i);
22596		if (depth < 0)
22597			return depth;
22598		bpf_patch_call_args(insn, depth);
22599	}
22600	err = 0;
22601#endif
22602	return err;
22603}
22604
22605/* replace a generic kfunc with a specialized version if necessary */
22606static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc, int insn_idx)
22607{
22608	struct bpf_prog *prog = env->prog;
22609	bool seen_direct_write;
22610	void *xdp_kfunc;
22611	bool is_rdonly;
22612	u32 func_id = desc->func_id;
22613	u16 offset = desc->offset;
22614	unsigned long addr = desc->addr;
22615
22616	if (offset) /* return if module BTF is used */
22617		return 0;
22618
22619	if (bpf_dev_bound_kfunc_id(func_id)) {
22620		xdp_kfunc = bpf_dev_bound_resolve_kfunc(prog, func_id);
22621		if (xdp_kfunc)
22622			addr = (unsigned long)xdp_kfunc;
22623		/* fallback to default kfunc when not supported by netdev */
22624	} else if (func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
22625		seen_direct_write = env->seen_direct_write;
22626		is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
22627
22628		if (is_rdonly)
22629			addr = (unsigned long)bpf_dynptr_from_skb_rdonly;
22630
22631		/* restore env->seen_direct_write to its original value, since
22632		 * may_access_direct_pkt_data mutates it
22633		 */
22634		env->seen_direct_write = seen_direct_write;
22635	} else if (func_id == special_kfunc_list[KF_bpf_set_dentry_xattr]) {
22636		if (bpf_lsm_has_d_inode_locked(prog))
22637			addr = (unsigned long)bpf_set_dentry_xattr_locked;
22638	} else if (func_id == special_kfunc_list[KF_bpf_remove_dentry_xattr]) {
22639		if (bpf_lsm_has_d_inode_locked(prog))
22640			addr = (unsigned long)bpf_remove_dentry_xattr_locked;
22641	} else if (func_id == special_kfunc_list[KF_bpf_dynptr_from_file]) {
22642		if (!env->insn_aux_data[insn_idx].non_sleepable)
22643			addr = (unsigned long)bpf_dynptr_from_file_sleepable;
22644	}
22645	desc->addr = addr;
22646	return 0;
22647}
22648
22649static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux,
22650					    u16 struct_meta_reg,
22651					    u16 node_offset_reg,
22652					    struct bpf_insn *insn,
22653					    struct bpf_insn *insn_buf,
22654					    int *cnt)
22655{
22656	struct btf_struct_meta *kptr_struct_meta = insn_aux->kptr_struct_meta;
22657	struct bpf_insn addr[2] = { BPF_LD_IMM64(struct_meta_reg, (long)kptr_struct_meta) };
22658
22659	insn_buf[0] = addr[0];
22660	insn_buf[1] = addr[1];
22661	insn_buf[2] = BPF_MOV64_IMM(node_offset_reg, insn_aux->insert_off);
22662	insn_buf[3] = *insn;
22663	*cnt = 4;
22664}
22665
22666static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
22667			    struct bpf_insn *insn_buf, int insn_idx, int *cnt)
22668{
22669	struct bpf_kfunc_desc *desc;
22670	int err;
22671
22672	if (!insn->imm) {
22673		verbose(env, "invalid kernel function call not eliminated in verifier pass\n");
22674		return -EINVAL;
22675	}
22676
22677	*cnt = 0;
22678
22679	/* insn->imm has the btf func_id. Replace it with an offset relative to
22680	 * __bpf_call_base, unless the JIT needs to call functions that are
22681	 * further than 32 bits away (bpf_jit_supports_far_kfunc_call()).
22682	 */
22683	desc = find_kfunc_desc(env->prog, insn->imm, insn->off);
22684	if (!desc) {
22685		verifier_bug(env, "kernel function descriptor not found for func_id %u",
22686			     insn->imm);
22687		return -EFAULT;
22688	}
22689
22690	err = specialize_kfunc(env, desc, insn_idx);
22691	if (err)
22692		return err;
22693
22694	if (!bpf_jit_supports_far_kfunc_call())
22695		insn->imm = BPF_CALL_IMM(desc->addr);
22696	if (insn->off)
22697		return 0;
22698	if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl] ||
22699	    desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
22700		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
22701		struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) };
22702		u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size;
22703
22704		if (desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl] && kptr_struct_meta) {
22705			verifier_bug(env, "NULL kptr_struct_meta expected at insn_idx %d",
22706				     insn_idx);
22707			return -EFAULT;
22708		}
22709
22710		insn_buf[0] = BPF_MOV64_IMM(BPF_REG_1, obj_new_size);
22711		insn_buf[1] = addr[0];
22712		insn_buf[2] = addr[1];
22713		insn_buf[3] = *insn;
22714		*cnt = 4;
22715	} else if (desc->func_id == special_kfunc_list[KF_bpf_obj_drop_impl] ||
22716		   desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl] ||
22717		   desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) {
22718		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
22719		struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) };
22720
22721		if (desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl] && kptr_struct_meta) {
22722			verifier_bug(env, "NULL kptr_struct_meta expected at insn_idx %d",
22723				     insn_idx);
22724			return -EFAULT;
22725		}
22726
22727		if (desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl] &&
22728		    !kptr_struct_meta) {
22729			verifier_bug(env, "kptr_struct_meta expected at insn_idx %d",
22730				     insn_idx);
22731			return -EFAULT;
22732		}
22733
22734		insn_buf[0] = addr[0];
22735		insn_buf[1] = addr[1];
22736		insn_buf[2] = *insn;
22737		*cnt = 3;
22738	} else if (desc->func_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||
22739		   desc->func_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
22740		   desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
22741		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
22742		int struct_meta_reg = BPF_REG_3;
22743		int node_offset_reg = BPF_REG_4;
22744
22745		/* rbtree_add has extra 'less' arg, so args-to-fixup are in diff regs */
22746		if (desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
22747			struct_meta_reg = BPF_REG_4;
22748			node_offset_reg = BPF_REG_5;
22749		}
22750
22751		if (!kptr_struct_meta) {
22752			verifier_bug(env, "kptr_struct_meta expected at insn_idx %d",
22753				     insn_idx);
22754			return -EFAULT;
22755		}
22756
22757		__fixup_collection_insert_kfunc(&env->insn_aux_data[insn_idx], struct_meta_reg,
22758						node_offset_reg, insn, insn_buf, cnt);
22759	} else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||
22760		   desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
22761		insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
22762		*cnt = 1;
22763	}
22764
22765	if (env->insn_aux_data[insn_idx].arg_prog) {
22766		u32 regno = env->insn_aux_data[insn_idx].arg_prog;
22767		struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(regno, (long)env->prog->aux) };
22768		int idx = *cnt;
22769
22770		insn_buf[idx++] = ld_addrs[0];
22771		insn_buf[idx++] = ld_addrs[1];
22772		insn_buf[idx++] = *insn;
22773		*cnt = idx;
22774	}
22775	return 0;
22776}
22777
22778/* The function requires that first instruction in 'patch' is insnsi[prog->len - 1] */
22779static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len)
22780{
22781	struct bpf_subprog_info *info = env->subprog_info;
22782	int cnt = env->subprog_cnt;
22783	struct bpf_prog *prog;
22784
22785	/* We only reserve one slot for hidden subprogs in subprog_info. */
22786	if (env->hidden_subprog_cnt) {
22787		verifier_bug(env, "only one hidden subprog supported");
22788		return -EFAULT;
22789	}
22790	/* We're not patching any existing instruction, just appending the new
22791	 * ones for the hidden subprog. Hence all of the adjustment operations
22792	 * in bpf_patch_insn_data are no-ops.
22793	 */
22794	prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len);
22795	if (!prog)
22796		return -ENOMEM;
22797	env->prog = prog;
22798	info[cnt + 1].start = info[cnt].start;
22799	info[cnt].start = prog->len - len + 1;
22800	env->subprog_cnt++;
22801	env->hidden_subprog_cnt++;
22802	return 0;
22803}
22804
22805/* Do various post-verification rewrites in a single program pass.
22806 * These rewrites simplify JIT and interpreter implementations.
22807 */
22808static int do_misc_fixups(struct bpf_verifier_env *env)
22809{
22810	struct bpf_prog *prog = env->prog;
22811	enum bpf_attach_type eatype = prog->expected_attach_type;
22812	enum bpf_prog_type prog_type = resolve_prog_type(prog);
22813	struct bpf_insn *insn = prog->insnsi;
22814	const struct bpf_func_proto *fn;
22815	const int insn_cnt = prog->len;
22816	const struct bpf_map_ops *ops;
22817	struct bpf_insn_aux_data *aux;
22818	struct bpf_insn *insn_buf = env->insn_buf;
22819	struct bpf_prog *new_prog;
22820	struct bpf_map *map_ptr;
22821	int i, ret, cnt, delta = 0, cur_subprog = 0;
22822	struct bpf_subprog_info *subprogs = env->subprog_info;
22823	u16 stack_depth = subprogs[cur_subprog].stack_depth;
22824	u16 stack_depth_extra = 0;
22825
22826	if (env->seen_exception && !env->exception_callback_subprog) {
22827		struct bpf_insn *patch = insn_buf;
22828
22829		*patch++ = env->prog->insnsi[insn_cnt - 1];
22830		*patch++ = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
22831		*patch++ = BPF_EXIT_INSN();
22832		ret = add_hidden_subprog(env, insn_buf, patch - insn_buf);
22833		if (ret < 0)
22834			return ret;
22835		prog = env->prog;
22836		insn = prog->insnsi;
22837
22838		env->exception_callback_subprog = env->subprog_cnt - 1;
22839		/* Don't update insn_cnt, as add_hidden_subprog always appends insns */
22840		mark_subprog_exc_cb(env, env->exception_callback_subprog);
22841	}
22842
22843	for (i = 0; i < insn_cnt;) {
22844		if (insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) && insn->imm) {
22845			if ((insn->off == BPF_ADDR_SPACE_CAST && insn->imm == 1) ||
22846			    (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) {
22847				/* convert to 32-bit mov that clears upper 32-bit */
22848				insn->code = BPF_ALU | BPF_MOV | BPF_X;
22849				/* clear off and imm, so it's a normal 'wX = wY' from JIT pov */
22850				insn->off = 0;
22851				insn->imm = 0;
22852			} /* cast from as(0) to as(1) should be handled by JIT */
22853			goto next_insn;
22854		}
22855
22856		if (env->insn_aux_data[i + delta].needs_zext)
22857			/* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */
22858			insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code);
22859
22860		/* Make sdiv/smod divide-by-minus-one exceptions impossible. */
22861		if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) ||
22862		     insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) ||
22863		     insn->code == (BPF_ALU | BPF_MOD | BPF_K) ||
22864		     insn->code == (BPF_ALU | BPF_DIV | BPF_K)) &&
22865		    insn->off == 1 && insn->imm == -1) {
22866			bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
22867			bool isdiv = BPF_OP(insn->code) == BPF_DIV;
22868			struct bpf_insn *patch = insn_buf;
22869
22870			if (isdiv)
22871				*patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) |
22872							BPF_NEG | BPF_K, insn->dst_reg,
22873							0, 0, 0);
22874			else
22875				*patch++ = BPF_MOV32_IMM(insn->dst_reg, 0);
22876
22877			cnt = patch - insn_buf;
22878
22879			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
22880			if (!new_prog)
22881				return -ENOMEM;
22882
22883			delta    += cnt - 1;
22884			env->prog = prog = new_prog;
22885			insn      = new_prog->insnsi + i + delta;
22886			goto next_insn;
22887		}
22888
22889		/* Make divide-by-zero and divide-by-minus-one exceptions impossible. */
22890		if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) ||
22891		    insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
22892		    insn->code == (BPF_ALU | BPF_MOD | BPF_X) ||
22893		    insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
22894			bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
22895			bool isdiv = BPF_OP(insn->code) == BPF_DIV;
22896			bool is_sdiv = isdiv && insn->off == 1;
22897			bool is_smod = !isdiv && insn->off == 1;
22898			struct bpf_insn *patch = insn_buf;
22899
22900			if (is_sdiv) {
22901				/* [R,W]x sdiv 0 -> 0
22902				 * LLONG_MIN sdiv -1 -> LLONG_MIN
22903				 * INT_MIN sdiv -1 -> INT_MIN
22904				 */
22905				*patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg);
22906				*patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) |
22907							BPF_ADD | BPF_K, BPF_REG_AX,
22908							0, 0, 1);
22909				*patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
22910							BPF_JGT | BPF_K, BPF_REG_AX,
22911							0, 4, 1);
22912				*patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
22913							BPF_JEQ | BPF_K, BPF_REG_AX,
22914							0, 1, 0);
22915				*patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) |
22916							BPF_MOV | BPF_K, insn->dst_reg,
22917							0, 0, 0);
22918				/* BPF_NEG(LLONG_MIN) == -LLONG_MIN == LLONG_MIN */
22919				*patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) |
22920							BPF_NEG | BPF_K, insn->dst_reg,
22921							0, 0, 0);
22922				*patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
22923				*patch++ = *insn;
22924				cnt = patch - insn_buf;
22925			} else if (is_smod) {
22926				/* [R,W]x mod 0 -> [R,W]x */
22927				/* [R,W]x mod -1 -> 0 */
22928				*patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg);
22929				*patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) |
22930							BPF_ADD | BPF_K, BPF_REG_AX,
22931							0, 0, 1);
22932				*patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
22933							BPF_JGT | BPF_K, BPF_REG_AX,
22934							0, 3, 1);
22935				*patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
22936							BPF_JEQ | BPF_K, BPF_REG_AX,
22937							0, 3 + (is64 ? 0 : 1), 1);
22938				*patch++ = BPF_MOV32_IMM(insn->dst_reg, 0);
22939				*patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
22940				*patch++ = *insn;
22941
22942				if (!is64) {
22943					*patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
22944					*patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg);
22945				}
22946				cnt = patch - insn_buf;
22947			} else if (isdiv) {
22948				/* [R,W]x div 0 -> 0 */
22949				*patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
22950							BPF_JNE | BPF_K, insn->src_reg,
22951							0, 2, 0);
22952				*patch++ = BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg);
22953				*patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
22954				*patch++ = *insn;
22955				cnt = patch - insn_buf;
22956			} else {
22957				/* [R,W]x mod 0 -> [R,W]x */
22958				*patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
22959							BPF_JEQ | BPF_K, insn->src_reg,
22960							0, 1 + (is64 ? 0 : 1), 0);
22961				*patch++ = *insn;
22962
22963				if (!is64) {
22964					*patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
22965					*patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg);
22966				}
22967				cnt = patch - insn_buf;
22968			}
22969
22970			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
22971			if (!new_prog)
22972				return -ENOMEM;
22973
22974			delta    += cnt - 1;
22975			env->prog = prog = new_prog;
22976			insn      = new_prog->insnsi + i + delta;
22977			goto next_insn;
22978		}
22979
22980		/* Make it impossible to de-reference a userspace address */
22981		if (BPF_CLASS(insn->code) == BPF_LDX &&
22982		    (BPF_MODE(insn->code) == BPF_PROBE_MEM ||
22983		     BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) {
22984			struct bpf_insn *patch = insn_buf;
22985			u64 uaddress_limit = bpf_arch_uaddress_limit();
22986
22987			if (!uaddress_limit)
22988				goto next_insn;
22989
22990			*patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg);
22991			if (insn->off)
22992				*patch++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, insn->off);
22993			*patch++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_AX, 32);
22994			*patch++ = BPF_JMP_IMM(BPF_JLE, BPF_REG_AX, uaddress_limit >> 32, 2);
22995			*patch++ = *insn;
22996			*patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
22997			*patch++ = BPF_MOV64_IMM(insn->dst_reg, 0);
22998
22999			cnt = patch - insn_buf;
23000			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
23001			if (!new_prog)
23002				return -ENOMEM;
23003
23004			delta    += cnt - 1;
23005			env->prog = prog = new_prog;
23006			insn      = new_prog->insnsi + i + delta;
23007			goto next_insn;
23008		}
23009
23010		/* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */
23011		if (BPF_CLASS(insn->code) == BPF_LD &&
23012		    (BPF_MODE(insn->code) == BPF_ABS ||
23013		     BPF_MODE(insn->code) == BPF_IND)) {
23014			cnt = env->ops->gen_ld_abs(insn, insn_buf);
23015			if (cnt == 0 || cnt >= INSN_BUF_SIZE) {
23016				verifier_bug(env, "%d insns generated for ld_abs", cnt);
23017				return -EFAULT;
23018			}
23019
23020			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
23021			if (!new_prog)
23022				return -ENOMEM;
23023
23024			delta    += cnt - 1;
23025			env->prog = prog = new_prog;
23026			insn      = new_prog->insnsi + i + delta;
23027			goto next_insn;
23028		}
23029
23030		/* Rewrite pointer arithmetic to mitigate speculation attacks. */
23031		if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) ||
23032		    insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) {
23033			const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X;
23034			const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X;
23035			struct bpf_insn *patch = insn_buf;
23036			bool issrc, isneg, isimm;
23037			u32 off_reg;
23038
23039			aux = &env->insn_aux_data[i + delta];
23040			if (!aux->alu_state ||
23041			    aux->alu_state == BPF_ALU_NON_POINTER)
23042				goto next_insn;
23043
23044			isneg = aux->alu_state & BPF_ALU_NEG_VALUE;
23045			issrc = (aux->alu_state & BPF_ALU_SANITIZE) ==
23046				BPF_ALU_SANITIZE_SRC;
23047			isimm = aux->alu_state & BPF_ALU_IMMEDIATE;
23048
23049			off_reg = issrc ? insn->src_reg : insn->dst_reg;
23050			if (isimm) {
23051				*patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
23052			} else {
23053				if (isneg)
23054					*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
23055				*patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
23056				*patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg);
23057				*patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg);
23058				*patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0);
23059				*patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63);
23060				*patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg);
23061			}
23062			if (!issrc)
23063				*patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg);
23064			insn->src_reg = BPF_REG_AX;
23065			if (isneg)
23066				insn->code = insn->code == code_add ?
23067					     code_sub : code_add;
23068			*patch++ = *insn;
23069			if (issrc && isneg && !isimm)
23070				*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
23071			cnt = patch - insn_buf;
23072
23073			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
23074			if (!new_prog)
23075				return -ENOMEM;
23076
23077			delta    += cnt - 1;
23078			env->prog = prog = new_prog;
23079			insn      = new_prog->insnsi + i + delta;
23080			goto next_insn;
23081		}
23082
23083		if (is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) {
23084			int stack_off_cnt = -stack_depth - 16;
23085
23086			/*
23087			 * Two 8 byte slots, depth-16 stores the count, and
23088			 * depth-8 stores the start timestamp of the loop.
23089			 *
23090			 * The starting value of count is BPF_MAX_TIMED_LOOPS
23091			 * (0xffff).  Every iteration loads it and subs it by 1,
23092			 * until the value becomes 0 in AX (thus, 1 in stack),
23093			 * after which we call arch_bpf_timed_may_goto, which
23094			 * either sets AX to 0xffff to keep looping, or to 0
23095			 * upon timeout. AX is then stored into the stack. In
23096			 * the next iteration, we either see 0 and break out, or
23097			 * continue iterating until the next time value is 0
23098			 * after subtraction, rinse and repeat.
23099			 */
23100			stack_depth_extra = 16;
23101			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off_cnt);
23102			if (insn->off >= 0)
23103				insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 5);
23104			else
23105				insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1);
23106			insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1);
23107			insn_buf[3] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2);
23108			/*
23109			 * AX is used as an argument to pass in stack_off_cnt
23110			 * (to add to r10/fp), and also as the return value of
23111			 * the call to arch_bpf_timed_may_goto.
23112			 */
23113			insn_buf[4] = BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt);
23114			insn_buf[5] = BPF_EMIT_CALL(arch_bpf_timed_may_goto);
23115			insn_buf[6] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt);
23116			cnt = 7;
23117
23118			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
23119			if (!new_prog)
23120				return -ENOMEM;
23121
23122			delta += cnt - 1;
23123			env->prog = prog = new_prog;
23124			insn = new_prog->insnsi + i + delta;
23125			goto next_insn;
23126		} else if (is_may_goto_insn(insn)) {
23127			int stack_off = -stack_depth - 8;
23128
23129			stack_depth_extra = 8;
23130			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off);
23131			if (insn->off >= 0)
23132				insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2);
23133			else
23134				insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1);
23135			insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1);
23136			insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off);
23137			cnt = 4;
23138
23139			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
23140			if (!new_prog)
23141				return -ENOMEM;
23142
23143			delta += cnt - 1;
23144			env->prog = prog = new_prog;
23145			insn = new_prog->insnsi + i + delta;
23146			goto next_insn;
23147		}
23148
23149		if (insn->code != (BPF_JMP | BPF_CALL))
23150			goto next_insn;
23151		if (insn->src_reg == BPF_PSEUDO_CALL)
23152			goto next_insn;
23153		if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
23154			ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt);
23155			if (ret)
23156				return ret;
23157			if (cnt == 0)
23158				goto next_insn;
23159
23160			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
23161			if (!new_prog)
23162				return -ENOMEM;
23163
23164			delta	 += cnt - 1;
23165			env->prog = prog = new_prog;
23166			insn	  = new_prog->insnsi + i + delta;
23167			goto next_insn;
23168		}
23169
23170		/* Skip inlining the helper call if the JIT does it. */
23171		if (bpf_jit_inlines_helper_call(insn->imm))
23172			goto next_insn;
23173
23174		if (insn->imm == BPF_FUNC_get_route_realm)
23175			prog->dst_needed = 1;
23176		if (insn->imm == BPF_FUNC_get_prandom_u32)
23177			bpf_user_rnd_init_once();
23178		if (insn->imm == BPF_FUNC_override_return)
23179			prog->kprobe_override = 1;
23180		if (insn->imm == BPF_FUNC_tail_call) {
23181			/* If we tail call into other programs, we
23182			 * cannot make any assumptions since they can
23183			 * be replaced dynamically during runtime in
23184			 * the program array.
23185			 */
23186			prog->cb_access = 1;
23187			if (!allow_tail_call_in_subprogs(env))
23188				prog->aux->stack_depth = MAX_BPF_STACK;
23189			prog->aux->max_pkt_offset = MAX_PACKET_OFF;
23190
23191			/* mark bpf_tail_call as different opcode to avoid
23192			 * conditional branch in the interpreter for every normal
23193			 * call and to prevent accidental JITing by JIT compiler
23194			 * that doesn't support bpf_tail_call yet
23195			 */
23196			insn->imm = 0;
23197			insn->code = BPF_JMP | BPF_TAIL_CALL;
23198
23199			aux = &env->insn_aux_data[i + delta];
23200			if (env->bpf_capable && !prog->blinding_requested &&
23201			    prog->jit_requested &&
23202			    !bpf_map_key_poisoned(aux) &&
23203			    !bpf_map_ptr_poisoned(aux) &&
23204			    !bpf_map_ptr_unpriv(aux)) {
23205				struct bpf_jit_poke_descriptor desc = {
23206					.reason = BPF_POKE_REASON_TAIL_CALL,
23207					.tail_call.map = aux->map_ptr_state.map_ptr,
23208					.tail_call.key = bpf_map_key_immediate(aux),
23209					.insn_idx = i + delta,
23210				};
23211
23212				ret = bpf_jit_add_poke_descriptor(prog, &desc);
23213				if (ret < 0) {
23214					verbose(env, "adding tail call poke descriptor failed\n");
23215					return ret;
23216				}
23217
23218				insn->imm = ret + 1;
23219				goto next_insn;
23220			}
23221
23222			if (!bpf_map_ptr_unpriv(aux))
23223				goto next_insn;
23224
23225			/* instead of changing every JIT dealing with tail_call
23226			 * emit two extra insns:
23227			 * if (index >= max_entries) goto out;
23228			 * index &= array->index_mask;
23229			 * to avoid out-of-bounds cpu speculation
23230			 */
23231			if (bpf_map_ptr_poisoned(aux)) {
23232				verbose(env, "tail_call abusing map_ptr\n");
23233				return -EINVAL;
23234			}
23235
23236			map_ptr = aux->map_ptr_state.map_ptr;
23237			insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
23238						  map_ptr->max_entries, 2);
23239			insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3,
23240						    container_of(map_ptr,
23241								 struct bpf_array,
23242								 map)->index_mask);
23243			insn_buf[2] = *insn;
23244			cnt = 3;
23245			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
23246			if (!new_prog)
23247				return -ENOMEM;
23248
23249			delta    += cnt - 1;
23250			env->prog = prog = new_prog;
23251			insn      = new_prog->insnsi + i + delta;
23252			goto next_insn;
23253		}
23254
23255		if (insn->imm == BPF_FUNC_timer_set_callback) {
23256			/* The verifier will process callback_fn as many times as necessary
23257			 * with different maps and the register states prepared by
23258			 * set_timer_callback_state will be accurate.
23259			 *
23260			 * The following use case is valid:
23261			 *   map1 is shared by prog1, prog2, prog3.
23262			 *   prog1 calls bpf_timer_init for some map1 elements
23263			 *   prog2 calls bpf_timer_set_callback for some map1 elements.
23264			 *     Those that were not bpf_timer_init-ed will return -EINVAL.
23265			 *   prog3 calls bpf_timer_start for some map1 elements.
23266			 *     Those that were not both bpf_timer_init-ed and
23267			 *     bpf_timer_set_callback-ed will return -EINVAL.
23268			 */
23269			struct bpf_insn ld_addrs[2] = {
23270				BPF_LD_IMM64(BPF_REG_3, (long)prog->aux),
23271			};
23272
23273			insn_buf[0] = ld_addrs[0];
23274			insn_buf[1] = ld_addrs[1];
23275			insn_buf[2] = *insn;
23276			cnt = 3;
23277
23278			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
23279			if (!new_prog)
23280				return -ENOMEM;
23281
23282			delta    += cnt - 1;
23283			env->prog = prog = new_prog;
23284			insn      = new_prog->insnsi + i + delta;
23285			goto patch_call_imm;
23286		}
23287
23288		if (is_storage_get_function(insn->imm)) {
23289			if (env->insn_aux_data[i + delta].non_sleepable)
23290				insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_ATOMIC);
23291			else
23292				insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_KERNEL);
23293			insn_buf[1] = *insn;
23294			cnt = 2;
23295
23296			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
23297			if (!new_prog)
23298				return -ENOMEM;
23299
23300			delta += cnt - 1;
23301			env->prog = prog = new_prog;
23302			insn = new_prog->insnsi + i + delta;
23303			goto patch_call_imm;
23304		}
23305
23306		/* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */
23307		if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) {
23308			/* patch with 'r1 = *(u64 *)(r1 + 0)' since for percpu data,
23309			 * bpf_mem_alloc() returns a ptr to the percpu data ptr.
23310			 */
23311			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0);
23312			insn_buf[1] = *insn;
23313			cnt = 2;
23314
23315			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
23316			if (!new_prog)
23317				return -ENOMEM;
23318
23319			delta += cnt - 1;
23320			env->prog = prog = new_prog;
23321			insn = new_prog->insnsi + i + delta;
23322			goto patch_call_imm;
23323		}
23324
23325		/* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup
23326		 * and other inlining handlers are currently limited to 64 bit
23327		 * only.
23328		 */
23329		if (prog->jit_requested && BITS_PER_LONG == 64 &&
23330		    (insn->imm == BPF_FUNC_map_lookup_elem ||
23331		     insn->imm == BPF_FUNC_map_update_elem ||
23332		     insn->imm == BPF_FUNC_map_delete_elem ||
23333		     insn->imm == BPF_FUNC_map_push_elem   ||
23334		     insn->imm == BPF_FUNC_map_pop_elem    ||
23335		     insn->imm == BPF_FUNC_map_peek_elem   ||
23336		     insn->imm == BPF_FUNC_redirect_map    ||
23337		     insn->imm == BPF_FUNC_for_each_map_elem ||
23338		     insn->imm == BPF_FUNC_map_lookup_percpu_elem)) {
23339			aux = &env->insn_aux_data[i + delta];
23340			if (bpf_map_ptr_poisoned(aux))
23341				goto patch_call_imm;
23342
23343			map_ptr = aux->map_ptr_state.map_ptr;
23344			ops = map_ptr->ops;
23345			if (insn->imm == BPF_FUNC_map_lookup_elem &&
23346			    ops->map_gen_lookup) {
23347				cnt = ops->map_gen_lookup(map_ptr, insn_buf);
23348				if (cnt == -EOPNOTSUPP)
23349					goto patch_map_ops_generic;
23350				if (cnt <= 0 || cnt >= INSN_BUF_SIZE) {
23351					verifier_bug(env, "%d insns generated for map lookup", cnt);
23352					return -EFAULT;
23353				}
23354
23355				new_prog = bpf_patch_insn_data(env, i + delta,
23356							       insn_buf, cnt);
23357				if (!new_prog)
23358					return -ENOMEM;
23359
23360				delta    += cnt - 1;
23361				env->prog = prog = new_prog;
23362				insn      = new_prog->insnsi + i + delta;
23363				goto next_insn;
23364			}
23365
23366			BUILD_BUG_ON(!__same_type(ops->map_lookup_elem,
23367				     (void *(*)(struct bpf_map *map, void *key))NULL));
23368			BUILD_BUG_ON(!__same_type(ops->map_delete_elem,
23369				     (long (*)(struct bpf_map *map, void *key))NULL));
23370			BUILD_BUG_ON(!__same_type(ops->map_update_elem,
23371				     (long (*)(struct bpf_map *map, void *key, void *value,
23372					      u64 flags))NULL));
23373			BUILD_BUG_ON(!__same_type(ops->map_push_elem,
23374				     (long (*)(struct bpf_map *map, void *value,
23375					      u64 flags))NULL));
23376			BUILD_BUG_ON(!__same_type(ops->map_pop_elem,
23377				     (long (*)(struct bpf_map *map, void *value))NULL));
23378			BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
23379				     (long (*)(struct bpf_map *map, void *value))NULL));
23380			BUILD_BUG_ON(!__same_type(ops->map_redirect,
23381				     (long (*)(struct bpf_map *map, u64 index, u64 flags))NULL));
23382			BUILD_BUG_ON(!__same_type(ops->map_for_each_callback,
23383				     (long (*)(struct bpf_map *map,
23384					      bpf_callback_t callback_fn,
23385					      void *callback_ctx,
23386					      u64 flags))NULL));
23387			BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem,
23388				     (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL));
23389
23390patch_map_ops_generic:
23391			switch (insn->imm) {
23392			case BPF_FUNC_map_lookup_elem:
23393				insn->imm = BPF_CALL_IMM(ops->map_lookup_elem);
23394				goto next_insn;
23395			case BPF_FUNC_map_update_elem:
23396				insn->imm = BPF_CALL_IMM(ops->map_update_elem);
23397				goto next_insn;
23398			case BPF_FUNC_map_delete_elem:
23399				insn->imm = BPF_CALL_IMM(ops->map_delete_elem);
23400				goto next_insn;
23401			case BPF_FUNC_map_push_elem:
23402				insn->imm = BPF_CALL_IMM(ops->map_push_elem);
23403				goto next_insn;
23404			case BPF_FUNC_map_pop_elem:
23405				insn->imm = BPF_CALL_IMM(ops->map_pop_elem);
23406				goto next_insn;
23407			case BPF_FUNC_map_peek_elem:
23408				insn->imm = BPF_CALL_IMM(ops->map_peek_elem);
23409				goto next_insn;
23410			case BPF_FUNC_redirect_map:
23411				insn->imm = BPF_CALL_IMM(ops->map_redirect);
23412				goto next_insn;
23413			case BPF_FUNC_for_each_map_elem:
23414				insn->imm = BPF_CALL_IMM(ops->map_for_each_callback);
23415				goto next_insn;
23416			case BPF_FUNC_map_lookup_percpu_elem:
23417				insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem);
23418				goto next_insn;
23419			}
23420
23421			goto patch_call_imm;
23422		}
23423
23424		/* Implement bpf_jiffies64 inline. */
23425		if (prog->jit_requested && BITS_PER_LONG == 64 &&
23426		    insn->imm == BPF_FUNC_jiffies64) {
23427			struct bpf_insn ld_jiffies_addr[2] = {
23428				BPF_LD_IMM64(BPF_REG_0,
23429					     (unsigned long)&jiffies),
23430			};
23431
23432			insn_buf[0] = ld_jiffies_addr[0];
23433			insn_buf[1] = ld_jiffies_addr[1];
23434			insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0,
23435						  BPF_REG_0, 0);
23436			cnt = 3;
23437
23438			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,
23439						       cnt);
23440			if (!new_prog)
23441				return -ENOMEM;
23442
23443			delta    += cnt - 1;
23444			env->prog = prog = new_prog;
23445			insn      = new_prog->insnsi + i + delta;
23446			goto next_insn;
23447		}
23448
23449#if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
23450		/* Implement bpf_get_smp_processor_id() inline. */
23451		if (insn->imm == BPF_FUNC_get_smp_processor_id &&
23452		    verifier_inlines_helper_call(env, insn->imm)) {
23453			/* BPF_FUNC_get_smp_processor_id inlining is an
23454			 * optimization, so if cpu_number is ever
23455			 * changed in some incompatible and hard to support
23456			 * way, it's fine to back out this inlining logic
23457			 */
23458#ifdef CONFIG_SMP
23459			insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&cpu_number);
23460			insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);
23461			insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0);
23462			cnt = 3;
23463#else
23464			insn_buf[0] = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0);
23465			cnt = 1;
23466#endif
23467			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
23468			if (!new_prog)
23469				return -ENOMEM;
23470
23471			delta    += cnt - 1;
23472			env->prog = prog = new_prog;
23473			insn      = new_prog->insnsi + i + delta;
23474			goto next_insn;
23475		}
23476#endif
23477		/* Implement bpf_get_func_arg inline. */
23478		if (prog_type == BPF_PROG_TYPE_TRACING &&
23479		    insn->imm == BPF_FUNC_get_func_arg) {
23480			/* Load nr_args from ctx - 8 */
23481			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
23482			insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6);
23483			insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3);
23484			insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1);
23485			insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0);
23486			insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
23487			insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0);
23488			insn_buf[7] = BPF_JMP_A(1);
23489			insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
23490			cnt = 9;
23491
23492			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
23493			if (!new_prog)
23494				return -ENOMEM;
23495
23496			delta    += cnt - 1;
23497			env->prog = prog = new_prog;
23498			insn      = new_prog->insnsi + i + delta;
23499			goto next_insn;
23500		}
23501
23502		/* Implement bpf_get_func_ret inline. */
23503		if (prog_type == BPF_PROG_TYPE_TRACING &&
23504		    insn->imm == BPF_FUNC_get_func_ret) {
23505			if (eatype == BPF_TRACE_FEXIT ||
23506			    eatype == BPF_MODIFY_RETURN) {
23507				/* Load nr_args from ctx - 8 */
23508				insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
23509				insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3);
23510				insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1);
23511				insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
23512				insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0);
23513				insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0);
23514				cnt = 6;
23515			} else {
23516				insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP);
23517				cnt = 1;
23518			}
23519
23520			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
23521			if (!new_prog)
23522				return -ENOMEM;
23523
23524			delta    += cnt - 1;
23525			env->prog = prog = new_prog;
23526			insn      = new_prog->insnsi + i + delta;
23527			goto next_insn;
23528		}
23529
23530		/* Implement get_func_arg_cnt inline. */
23531		if (prog_type == BPF_PROG_TYPE_TRACING &&
23532		    insn->imm == BPF_FUNC_get_func_arg_cnt) {
23533			/* Load nr_args from ctx - 8 */
23534			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
23535
23536			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
23537			if (!new_prog)
23538				return -ENOMEM;
23539
23540			env->prog = prog = new_prog;
23541			insn      = new_prog->insnsi + i + delta;
23542			goto next_insn;
23543		}
23544
23545		/* Implement bpf_get_func_ip inline. */
23546		if (prog_type == BPF_PROG_TYPE_TRACING &&
23547		    insn->imm == BPF_FUNC_get_func_ip) {
23548			/* Load IP address from ctx - 16 */
23549			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16);
23550
23551			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
23552			if (!new_prog)
23553				return -ENOMEM;
23554
23555			env->prog = prog = new_prog;
23556			insn      = new_prog->insnsi + i + delta;
23557			goto next_insn;
23558		}
23559
23560		/* Implement bpf_get_branch_snapshot inline. */
23561		if (IS_ENABLED(CONFIG_PERF_EVENTS) &&
23562		    prog->jit_requested && BITS_PER_LONG == 64 &&
23563		    insn->imm == BPF_FUNC_get_branch_snapshot) {
23564			/* We are dealing with the following func protos:
23565			 * u64 bpf_get_branch_snapshot(void *buf, u32 size, u64 flags);
23566			 * int perf_snapshot_branch_stack(struct perf_branch_entry *entries, u32 cnt);
23567			 */
23568			const u32 br_entry_size = sizeof(struct perf_branch_entry);
23569
23570			/* struct perf_branch_entry is part of UAPI and is
23571			 * used as an array element, so extremely unlikely to
23572			 * ever grow or shrink
23573			 */
23574			BUILD_BUG_ON(br_entry_size != 24);
23575
23576			/* if (unlikely(flags)) return -EINVAL */
23577			insn_buf[0] = BPF_JMP_IMM(BPF_JNE, BPF_REG_3, 0, 7);
23578
23579			/* Transform size (bytes) into number of entries (cnt = size / 24).
23580			 * But to avoid expensive division instruction, we implement
23581			 * divide-by-3 through multiplication, followed by further
23582			 * division by 8 through 3-bit right shift.
23583			 * Refer to book "Hacker's Delight, 2nd ed." by Henry S. Warren, Jr.,
23584			 * p. 227, chapter "Unsigned Division by 3" for details and proofs.
23585			 *
23586			 * N / 3 <=> M * N / 2^33, where M = (2^33 + 1) / 3 = 0xaaaaaaab.
23587			 */
23588			insn_buf[1] = BPF_MOV32_IMM(BPF_REG_0, 0xaaaaaaab);
23589			insn_buf[2] = BPF_ALU64_REG(BPF_MUL, BPF_REG_2, BPF_REG_0);
23590			insn_buf[3] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_2, 36);
23591
23592			/* call perf_snapshot_branch_stack implementation */
23593			insn_buf[4] = BPF_EMIT_CALL(static_call_query(perf_snapshot_branch_stack));
23594			/* if (entry_cnt == 0) return -ENOENT */
23595			insn_buf[5] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 4);
23596			/* return entry_cnt * sizeof(struct perf_branch_entry) */
23597			insn_buf[6] = BPF_ALU32_IMM(BPF_MUL, BPF_REG_0, br_entry_size);
23598			insn_buf[7] = BPF_JMP_A(3);
23599			/* return -EINVAL; */
23600			insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
23601			insn_buf[9] = BPF_JMP_A(1);
23602			/* return -ENOENT; */
23603			insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT);
23604			cnt = 11;
23605
23606			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
23607			if (!new_prog)
23608				return -ENOMEM;
23609
23610			delta    += cnt - 1;
23611			env->prog = prog = new_prog;
23612			insn      = new_prog->insnsi + i + delta;
23613			goto next_insn;
23614		}
23615
23616		/* Implement bpf_kptr_xchg inline */
23617		if (prog->jit_requested && BITS_PER_LONG == 64 &&
23618		    insn->imm == BPF_FUNC_kptr_xchg &&
23619		    bpf_jit_supports_ptr_xchg()) {
23620			insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2);
23621			insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0);
23622			cnt = 2;
23623
23624			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
23625			if (!new_prog)
23626				return -ENOMEM;
23627
23628			delta    += cnt - 1;
23629			env->prog = prog = new_prog;
23630			insn      = new_prog->insnsi + i + delta;
23631			goto next_insn;
23632		}
23633patch_call_imm:
23634		fn = env->ops->get_func_proto(insn->imm, env->prog);
23635		/* all functions that have prototype and verifier allowed
23636		 * programs to call them, must be real in-kernel functions
23637		 */
23638		if (!fn->func) {
23639			verifier_bug(env,
23640				     "not inlined functions %s#%d is missing func",
23641				     func_id_name(insn->imm), insn->imm);
23642			return -EFAULT;
23643		}
23644		insn->imm = fn->func - __bpf_call_base;
23645next_insn:
23646		if (subprogs[cur_subprog + 1].start == i + delta + 1) {
23647			subprogs[cur_subprog].stack_depth += stack_depth_extra;
23648			subprogs[cur_subprog].stack_extra = stack_depth_extra;
23649
23650			stack_depth = subprogs[cur_subprog].stack_depth;
23651			if (stack_depth > MAX_BPF_STACK && !prog->jit_requested) {
23652				verbose(env, "stack size %d(extra %d) is too large\n",
23653					stack_depth, stack_depth_extra);
23654				return -EINVAL;
23655			}
23656			cur_subprog++;
23657			stack_depth = subprogs[cur_subprog].stack_depth;
23658			stack_depth_extra = 0;
23659		}
23660		i++;
23661		insn++;
23662	}
23663
23664	env->prog->aux->stack_depth = subprogs[0].stack_depth;
23665	for (i = 0; i < env->subprog_cnt; i++) {
23666		int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1;
23667		int subprog_start = subprogs[i].start;
23668		int stack_slots = subprogs[i].stack_extra / 8;
23669		int slots = delta, cnt = 0;
23670
23671		if (!stack_slots)
23672			continue;
23673		/* We need two slots in case timed may_goto is supported. */
23674		if (stack_slots > slots) {
23675			verifier_bug(env, "stack_slots supports may_goto only");
23676			return -EFAULT;
23677		}
23678
23679		stack_depth = subprogs[i].stack_depth;
23680		if (bpf_jit_supports_timed_may_goto()) {
23681			insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth,
23682						     BPF_MAX_TIMED_LOOPS);
23683			insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth + 8, 0);
23684		} else {
23685			/* Add ST insn to subprog prologue to init extra stack */
23686			insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth,
23687						     BPF_MAX_LOOPS);
23688		}
23689		/* Copy first actual insn to preserve it */
23690		insn_buf[cnt++] = env->prog->insnsi[subprog_start];
23691
23692		new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt);
23693		if (!new_prog)
23694			return -ENOMEM;
23695		env->prog = prog = new_prog;
23696		/*
23697		 * If may_goto is a first insn of a prog there could be a jmp
23698		 * insn that points to it, hence adjust all such jmps to point
23699		 * to insn after BPF_ST that inits may_goto count.
23700		 * Adjustment will succeed because bpf_patch_insn_data() didn't fail.
23701		 */
23702		WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta));
23703	}
23704
23705	/* Since poke tab is now finalized, publish aux to tracker. */
23706	for (i = 0; i < prog->aux->size_poke_tab; i++) {
23707		map_ptr = prog->aux->poke_tab[i].tail_call.map;
23708		if (!map_ptr->ops->map_poke_track ||
23709		    !map_ptr->ops->map_poke_untrack ||
23710		    !map_ptr->ops->map_poke_run) {
23711			verifier_bug(env, "poke tab is misconfigured");
23712			return -EFAULT;
23713		}
23714
23715		ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux);
23716		if (ret < 0) {
23717			verbose(env, "tracking tail call prog failed\n");
23718			return ret;
23719		}
23720	}
23721
23722	ret = sort_kfunc_descs_by_imm_off(env);
23723	if (ret)
23724		return ret;
23725
23726	return 0;
23727}
23728
23729static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env,
23730					int position,
23731					s32 stack_base,
23732					u32 callback_subprogno,
23733					u32 *total_cnt)
23734{
23735	s32 r6_offset = stack_base + 0 * BPF_REG_SIZE;
23736	s32 r7_offset = stack_base + 1 * BPF_REG_SIZE;
23737	s32 r8_offset = stack_base + 2 * BPF_REG_SIZE;
23738	int reg_loop_max = BPF_REG_6;
23739	int reg_loop_cnt = BPF_REG_7;
23740	int reg_loop_ctx = BPF_REG_8;
23741
23742	struct bpf_insn *insn_buf = env->insn_buf;
23743	struct bpf_prog *new_prog;
23744	u32 callback_start;
23745	u32 call_insn_offset;
23746	s32 callback_offset;
23747	u32 cnt = 0;
23748
23749	/* This represents an inlined version of bpf_iter.c:bpf_loop,
23750	 * be careful to modify this code in sync.
23751	 */
23752
23753	/* Return error and jump to the end of the patch if
23754	 * expected number of iterations is too big.
23755	 */
23756	insn_buf[cnt++] = BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2);
23757	insn_buf[cnt++] = BPF_MOV32_IMM(BPF_REG_0, -E2BIG);
23758	insn_buf[cnt++] = BPF_JMP_IMM(BPF_JA, 0, 0, 16);
23759	/* spill R6, R7, R8 to use these as loop vars */
23760	insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset);
23761	insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset);
23762	insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset);
23763	/* initialize loop vars */
23764	insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_max, BPF_REG_1);
23765	insn_buf[cnt++] = BPF_MOV32_IMM(reg_loop_cnt, 0);
23766	insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3);
23767	/* loop header,
23768	 * if reg_loop_cnt >= reg_loop_max skip the loop body
23769	 */
23770	insn_buf[cnt++] = BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5);
23771	/* callback call,
23772	 * correct callback offset would be set after patching
23773	 */
23774	insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt);
23775	insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx);
23776	insn_buf[cnt++] = BPF_CALL_REL(0);
23777	/* increment loop counter */
23778	insn_buf[cnt++] = BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1);
23779	/* jump to loop header if callback returned 0 */
23780	insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6);
23781	/* return value of bpf_loop,
23782	 * set R0 to the number of iterations
23783	 */
23784	insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt);
23785	/* restore original values of R6, R7, R8 */
23786	insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset);
23787	insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset);
23788	insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset);
23789
23790	*total_cnt = cnt;
23791	new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt);
23792	if (!new_prog)
23793		return new_prog;
23794
23795	/* callback start is known only after patching */
23796	callback_start = env->subprog_info[callback_subprogno].start;
23797	/* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */
23798	call_insn_offset = position + 12;
23799	callback_offset = callback_start - call_insn_offset - 1;
23800	new_prog->insnsi[call_insn_offset].imm = callback_offset;
23801
23802	return new_prog;
23803}
23804
23805static bool is_bpf_loop_call(struct bpf_insn *insn)
23806{
23807	return insn->code == (BPF_JMP | BPF_CALL) &&
23808		insn->src_reg == 0 &&
23809		insn->imm == BPF_FUNC_loop;
23810}
23811
23812/* For all sub-programs in the program (including main) check
23813 * insn_aux_data to see if there are bpf_loop calls that require
23814 * inlining. If such calls are found the calls are replaced with a
23815 * sequence of instructions produced by `inline_bpf_loop` function and
23816 * subprog stack_depth is increased by the size of 3 registers.
23817 * This stack space is used to spill values of the R6, R7, R8.  These
23818 * registers are used to store the loop bound, counter and context
23819 * variables.
23820 */
23821static int optimize_bpf_loop(struct bpf_verifier_env *env)
23822{
23823	struct bpf_subprog_info *subprogs = env->subprog_info;
23824	int i, cur_subprog = 0, cnt, delta = 0;
23825	struct bpf_insn *insn = env->prog->insnsi;
23826	int insn_cnt = env->prog->len;
23827	u16 stack_depth = subprogs[cur_subprog].stack_depth;
23828	u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth;
23829	u16 stack_depth_extra = 0;
23830
23831	for (i = 0; i < insn_cnt; i++, insn++) {
23832		struct bpf_loop_inline_state *inline_state =
23833			&env->insn_aux_data[i + delta].loop_inline_state;
23834
23835		if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) {
23836			struct bpf_prog *new_prog;
23837
23838			stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup;
23839			new_prog = inline_bpf_loop(env,
23840						   i + delta,
23841						   -(stack_depth + stack_depth_extra),
23842						   inline_state->callback_subprogno,
23843						   &cnt);
23844			if (!new_prog)
23845				return -ENOMEM;
23846
23847			delta     += cnt - 1;
23848			env->prog  = new_prog;
23849			insn       = new_prog->insnsi + i + delta;
23850		}
23851
23852		if (subprogs[cur_subprog + 1].start == i + delta + 1) {
23853			subprogs[cur_subprog].stack_depth += stack_depth_extra;
23854			cur_subprog++;
23855			stack_depth = subprogs[cur_subprog].stack_depth;
23856			stack_depth_roundup = round_up(stack_depth, 8) - stack_depth;
23857			stack_depth_extra = 0;
23858		}
23859	}
23860
23861	env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
23862
23863	return 0;
23864}
23865
23866/* Remove unnecessary spill/fill pairs, members of fastcall pattern,
23867 * adjust subprograms stack depth when possible.
23868 */
23869static int remove_fastcall_spills_fills(struct bpf_verifier_env *env)
23870{
23871	struct bpf_subprog_info *subprog = env->subprog_info;
23872	struct bpf_insn_aux_data *aux = env->insn_aux_data;
23873	struct bpf_insn *insn = env->prog->insnsi;
23874	int insn_cnt = env->prog->len;
23875	u32 spills_num;
23876	bool modified = false;
23877	int i, j;
23878
23879	for (i = 0; i < insn_cnt; i++, insn++) {
23880		if (aux[i].fastcall_spills_num > 0) {
23881			spills_num = aux[i].fastcall_spills_num;
23882			/* NOPs would be removed by opt_remove_nops() */
23883			for (j = 1; j <= spills_num; ++j) {
23884				*(insn - j) = NOP;
23885				*(insn + j) = NOP;
23886			}
23887			modified = true;
23888		}
23889		if ((subprog + 1)->start == i + 1) {
23890			if (modified && !subprog->keep_fastcall_stack)
23891				subprog->stack_depth = -subprog->fastcall_stack_off;
23892			subprog++;
23893			modified = false;
23894		}
23895	}
23896
23897	return 0;
23898}
23899
23900static void free_states(struct bpf_verifier_env *env)
23901{
23902	struct bpf_verifier_state_list *sl;
23903	struct list_head *head, *pos, *tmp;
23904	struct bpf_scc_info *info;
23905	int i, j;
23906
23907	free_verifier_state(env->cur_state, true);
23908	env->cur_state = NULL;
23909	while (!pop_stack(env, NULL, NULL, false));
23910
23911	list_for_each_safe(pos, tmp, &env->free_list) {
23912		sl = container_of(pos, struct bpf_verifier_state_list, node);
23913		free_verifier_state(&sl->state, false);
23914		kfree(sl);
23915	}
23916	INIT_LIST_HEAD(&env->free_list);
23917
23918	for (i = 0; i < env->scc_cnt; ++i) {
23919		info = env->scc_info[i];
23920		if (!info)
23921			continue;
23922		for (j = 0; j < info->num_visits; j++)
23923			free_backedges(&info->visits[j]);
23924		kvfree(info);
23925		env->scc_info[i] = NULL;
23926	}
23927
23928	if (!env->explored_states)
23929		return;
23930
23931	for (i = 0; i < state_htab_size(env); i++) {
23932		head = &env->explored_states[i];
23933
23934		list_for_each_safe(pos, tmp, head) {
23935			sl = container_of(pos, struct bpf_verifier_state_list, node);
23936			free_verifier_state(&sl->state, false);
23937			kfree(sl);
23938		}
23939		INIT_LIST_HEAD(&env->explored_states[i]);
23940	}
23941}
23942
23943static int do_check_common(struct bpf_verifier_env *env, int subprog)
23944{
23945	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
23946	struct bpf_subprog_info *sub = subprog_info(env, subprog);
23947	struct bpf_prog_aux *aux = env->prog->aux;
23948	struct bpf_verifier_state *state;
23949	struct bpf_reg_state *regs;
23950	int ret, i;
23951
23952	env->prev_linfo = NULL;
23953	env->pass_cnt++;
23954
23955	state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL_ACCOUNT);
23956	if (!state)
23957		return -ENOMEM;
23958	state->curframe = 0;
23959	state->speculative = false;
23960	state->branches = 1;
23961	state->in_sleepable = env->prog->sleepable;
23962	state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL_ACCOUNT);
23963	if (!state->frame[0]) {
23964		kfree(state);
23965		return -ENOMEM;
23966	}
23967	env->cur_state = state;
23968	init_func_state(env, state->frame[0],
23969			BPF_MAIN_FUNC /* callsite */,
23970			0 /* frameno */,
23971			subprog);
23972	state->first_insn_idx = env->subprog_info[subprog].start;
23973	state->last_insn_idx = -1;
23974
23975	regs = state->frame[state->curframe]->regs;
23976	if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) {
23977		const char *sub_name = subprog_name(env, subprog);
23978		struct bpf_subprog_arg_info *arg;
23979		struct bpf_reg_state *reg;
23980
23981		if (env->log.level & BPF_LOG_LEVEL)
23982			verbose(env, "Validating %s() func#%d...\n", sub_name, subprog);
23983		ret = btf_prepare_func_args(env, subprog);
23984		if (ret)
23985			goto out;
23986
23987		if (subprog_is_exc_cb(env, subprog)) {
23988			state->frame[0]->in_exception_callback_fn = true;
23989			/* We have already ensured that the callback returns an integer, just
23990			 * like all global subprogs. We need to determine it only has a single
23991			 * scalar argument.
23992			 */
23993			if (sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_ANYTHING) {
23994				verbose(env, "exception cb only supports single integer argument\n");
23995				ret = -EINVAL;
23996				goto out;
23997			}
23998		}
23999		for (i = BPF_REG_1; i <= sub->arg_cnt; i++) {
24000			arg = &sub->args[i - BPF_REG_1];
24001			reg = &regs[i];
24002
24003			if (arg->arg_type == ARG_PTR_TO_CTX) {
24004				reg->type = PTR_TO_CTX;
24005				mark_reg_known_zero(env, regs, i);
24006			} else if (arg->arg_type == ARG_ANYTHING) {
24007				reg->type = SCALAR_VALUE;
24008				mark_reg_unknown(env, regs, i);
24009			} else if (arg->arg_type == (ARG_PTR_TO_DYNPTR | MEM_RDONLY)) {
24010				/* assume unspecial LOCAL dynptr type */
24011				__mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen);
24012			} else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) {
24013				reg->type = PTR_TO_MEM;
24014				reg->type |= arg->arg_type &
24015					     (PTR_MAYBE_NULL | PTR_UNTRUSTED | MEM_RDONLY);
24016				mark_reg_known_zero(env, regs, i);
24017				reg->mem_size = arg->mem_size;
24018				if (arg->arg_type & PTR_MAYBE_NULL)
24019					reg->id = ++env->id_gen;
24020			} else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) {
24021				reg->type = PTR_TO_BTF_ID;
24022				if (arg->arg_type & PTR_MAYBE_NULL)
24023					reg->type |= PTR_MAYBE_NULL;
24024				if (arg->arg_type & PTR_UNTRUSTED)
24025					reg->type |= PTR_UNTRUSTED;
24026				if (arg->arg_type & PTR_TRUSTED)
24027					reg->type |= PTR_TRUSTED;
24028				mark_reg_known_zero(env, regs, i);
24029				reg->btf = bpf_get_btf_vmlinux(); /* can't fail at this point */
24030				reg->btf_id = arg->btf_id;
24031				reg->id = ++env->id_gen;
24032			} else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) {
24033				/* caller can pass either PTR_TO_ARENA or SCALAR */
24034				mark_reg_unknown(env, regs, i);
24035			} else {
24036				verifier_bug(env, "unhandled arg#%d type %d",
24037					     i - BPF_REG_1, arg->arg_type);
24038				ret = -EFAULT;
24039				goto out;
24040			}
24041		}
24042	} else {
24043		/* if main BPF program has associated BTF info, validate that
24044		 * it's matching expected signature, and otherwise mark BTF
24045		 * info for main program as unreliable
24046		 */
24047		if (env->prog->aux->func_info_aux) {
24048			ret = btf_prepare_func_args(env, 0);
24049			if (ret || sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_PTR_TO_CTX)
24050				env->prog->aux->func_info_aux[0].unreliable = true;
24051		}
24052
24053		/* 1st arg to a function */
24054		regs[BPF_REG_1].type = PTR_TO_CTX;
24055		mark_reg_known_zero(env, regs, BPF_REG_1);
24056	}
24057
24058	/* Acquire references for struct_ops program arguments tagged with "__ref" */
24059	if (!subprog && env->prog->type == BPF_PROG_TYPE_STRUCT_OPS) {
24060		for (i = 0; i < aux->ctx_arg_info_size; i++)
24061			aux->ctx_arg_info[i].ref_obj_id = aux->ctx_arg_info[i].refcounted ?
24062							  acquire_reference(env, 0) : 0;
24063	}
24064
24065	ret = do_check(env);
24066out:
24067	if (!ret && pop_log)
24068		bpf_vlog_reset(&env->log, 0);
24069	free_states(env);
24070	return ret;
24071}
24072
24073/* Lazily verify all global functions based on their BTF, if they are called
24074 * from main BPF program or any of subprograms transitively.
24075 * BPF global subprogs called from dead code are not validated.
24076 * All callable global functions must pass verification.
24077 * Otherwise the whole program is rejected.
24078 * Consider:
24079 * int bar(int);
24080 * int foo(int f)
24081 * {
24082 *    return bar(f);
24083 * }
24084 * int bar(int b)
24085 * {
24086 *    ...
24087 * }
24088 * foo() will be verified first for R1=any_scalar_value. During verification it
24089 * will be assumed that bar() already verified successfully and call to bar()
24090 * from foo() will be checked for type match only. Later bar() will be verified
24091 * independently to check that it's safe for R1=any_scalar_value.
24092 */
24093static int do_check_subprogs(struct bpf_verifier_env *env)
24094{
24095	struct bpf_prog_aux *aux = env->prog->aux;
24096	struct bpf_func_info_aux *sub_aux;
24097	int i, ret, new_cnt;
24098
24099	if (!aux->func_info)
24100		return 0;
24101
24102	/* exception callback is presumed to be always called */
24103	if (env->exception_callback_subprog)
24104		subprog_aux(env, env->exception_callback_subprog)->called = true;
24105
24106again:
24107	new_cnt = 0;
24108	for (i = 1; i < env->subprog_cnt; i++) {
24109		if (!subprog_is_global(env, i))
24110			continue;
24111
24112		sub_aux = subprog_aux(env, i);
24113		if (!sub_aux->called || sub_aux->verified)
24114			continue;
24115
24116		env->insn_idx = env->subprog_info[i].start;
24117		WARN_ON_ONCE(env->insn_idx == 0);
24118		ret = do_check_common(env, i);
24119		if (ret) {
24120			return ret;
24121		} else if (env->log.level & BPF_LOG_LEVEL) {
24122			verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n",
24123				i, subprog_name(env, i));
24124		}
24125
24126		/* We verified new global subprog, it might have called some
24127		 * more global subprogs that we haven't verified yet, so we
24128		 * need to do another pass over subprogs to verify those.
24129		 */
24130		sub_aux->verified = true;
24131		new_cnt++;
24132	}
24133
24134	/* We can't loop forever as we verify at least one global subprog on
24135	 * each pass.
24136	 */
24137	if (new_cnt)
24138		goto again;
24139
24140	return 0;
24141}
24142
24143static int do_check_main(struct bpf_verifier_env *env)
24144{
24145	int ret;
24146
24147	env->insn_idx = 0;
24148	ret = do_check_common(env, 0);
24149	if (!ret)
24150		env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
24151	return ret;
24152}
24153
24154
24155static void print_verification_stats(struct bpf_verifier_env *env)
24156{
24157	int i;
24158
24159	if (env->log.level & BPF_LOG_STATS) {
24160		verbose(env, "verification time %lld usec\n",
24161			div_u64(env->verification_time, 1000));
24162		verbose(env, "stack depth ");
24163		for (i = 0; i < env->subprog_cnt; i++) {
24164			u32 depth = env->subprog_info[i].stack_depth;
24165
24166			verbose(env, "%d", depth);
24167			if (i + 1 < env->subprog_cnt)
24168				verbose(env, "+");
24169		}
24170		verbose(env, "\n");
24171	}
24172	verbose(env, "processed %d insns (limit %d) max_states_per_insn %d "
24173		"total_states %d peak_states %d mark_read %d\n",
24174		env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS,
24175		env->max_states_per_insn, env->total_states,
24176		env->peak_states, env->longest_mark_read_walk);
24177}
24178
24179int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog,
24180			       const struct bpf_ctx_arg_aux *info, u32 cnt)
24181{
24182	prog->aux->ctx_arg_info = kmemdup_array(info, cnt, sizeof(*info), GFP_KERNEL_ACCOUNT);
24183	prog->aux->ctx_arg_info_size = cnt;
24184
24185	return prog->aux->ctx_arg_info ? 0 : -ENOMEM;
24186}
24187
24188static int check_struct_ops_btf_id(struct bpf_verifier_env *env)
24189{
24190	const struct btf_type *t, *func_proto;
24191	const struct bpf_struct_ops_desc *st_ops_desc;
24192	const struct bpf_struct_ops *st_ops;
24193	const struct btf_member *member;
24194	struct bpf_prog *prog = env->prog;
24195	bool has_refcounted_arg = false;
24196	u32 btf_id, member_idx, member_off;
24197	struct btf *btf;
24198	const char *mname;
24199	int i, err;
24200
24201	if (!prog->gpl_compatible) {
24202		verbose(env, "struct ops programs must have a GPL compatible license\n");
24203		return -EINVAL;
24204	}
24205
24206	if (!prog->aux->attach_btf_id)
24207		return -ENOTSUPP;
24208
24209	btf = prog->aux->attach_btf;
24210	if (btf_is_module(btf)) {
24211		/* Make sure st_ops is valid through the lifetime of env */
24212		env->attach_btf_mod = btf_try_get_module(btf);
24213		if (!env->attach_btf_mod) {
24214			verbose(env, "struct_ops module %s is not found\n",
24215				btf_get_name(btf));
24216			return -ENOTSUPP;
24217		}
24218	}
24219
24220	btf_id = prog->aux->attach_btf_id;
24221	st_ops_desc = bpf_struct_ops_find(btf, btf_id);
24222	if (!st_ops_desc) {
24223		verbose(env, "attach_btf_id %u is not a supported struct\n",
24224			btf_id);
24225		return -ENOTSUPP;
24226	}
24227	st_ops = st_ops_desc->st_ops;
24228
24229	t = st_ops_desc->type;
24230	member_idx = prog->expected_attach_type;
24231	if (member_idx >= btf_type_vlen(t)) {
24232		verbose(env, "attach to invalid member idx %u of struct %s\n",
24233			member_idx, st_ops->name);
24234		return -EINVAL;
24235	}
24236
24237	member = &btf_type_member(t)[member_idx];
24238	mname = btf_name_by_offset(btf, member->name_off);
24239	func_proto = btf_type_resolve_func_ptr(btf, member->type,
24240					       NULL);
24241	if (!func_proto) {
24242		verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n",
24243			mname, member_idx, st_ops->name);
24244		return -EINVAL;
24245	}
24246
24247	member_off = __btf_member_bit_offset(t, member) / 8;
24248	err = bpf_struct_ops_supported(st_ops, member_off);
24249	if (err) {
24250		verbose(env, "attach to unsupported member %s of struct %s\n",
24251			mname, st_ops->name);
24252		return err;
24253	}
24254
24255	if (st_ops->check_member) {
24256		err = st_ops->check_member(t, member, prog);
24257
24258		if (err) {
24259			verbose(env, "attach to unsupported member %s of struct %s\n",
24260				mname, st_ops->name);
24261			return err;
24262		}
24263	}
24264
24265	if (prog->aux->priv_stack_requested && !bpf_jit_supports_private_stack()) {
24266		verbose(env, "Private stack not supported by jit\n");
24267		return -EACCES;
24268	}
24269
24270	for (i = 0; i < st_ops_desc->arg_info[member_idx].cnt; i++) {
24271		if (st_ops_desc->arg_info[member_idx].info->refcounted) {
24272			has_refcounted_arg = true;
24273			break;
24274		}
24275	}
24276
24277	/* Tail call is not allowed for programs with refcounted arguments since we
24278	 * cannot guarantee that valid refcounted kptrs will be passed to the callee.
24279	 */
24280	for (i = 0; i < env->subprog_cnt; i++) {
24281		if (has_refcounted_arg && env->subprog_info[i].has_tail_call) {
24282			verbose(env, "program with __ref argument cannot tail call\n");
24283			return -EINVAL;
24284		}
24285	}
24286
24287	prog->aux->st_ops = st_ops;
24288	prog->aux->attach_st_ops_member_off = member_off;
24289
24290	prog->aux->attach_func_proto = func_proto;
24291	prog->aux->attach_func_name = mname;
24292	env->ops = st_ops->verifier_ops;
24293
24294	return bpf_prog_ctx_arg_info_init(prog, st_ops_desc->arg_info[member_idx].info,
24295					  st_ops_desc->arg_info[member_idx].cnt);
24296}
24297#define SECURITY_PREFIX "security_"
24298
24299static int check_attach_modify_return(unsigned long addr, const char *func_name)
24300{
24301	if (within_error_injection_list(addr) ||
24302	    !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1))
24303		return 0;
24304
24305	return -EINVAL;
24306}
24307
24308/* list of non-sleepable functions that are otherwise on
24309 * ALLOW_ERROR_INJECTION list
24310 */
24311BTF_SET_START(btf_non_sleepable_error_inject)
24312/* Three functions below can be called from sleepable and non-sleepable context.
24313 * Assume non-sleepable from bpf safety point of view.
24314 */
24315BTF_ID(func, __filemap_add_folio)
24316#ifdef CONFIG_FAIL_PAGE_ALLOC
24317BTF_ID(func, should_fail_alloc_page)
24318#endif
24319#ifdef CONFIG_FAILSLAB
24320BTF_ID(func, should_failslab)
24321#endif
24322BTF_SET_END(btf_non_sleepable_error_inject)
24323
24324static int check_non_sleepable_error_inject(u32 btf_id)
24325{
24326	return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id);
24327}
24328
24329int bpf_check_attach_target(struct bpf_verifier_log *log,
24330			    const struct bpf_prog *prog,
24331			    const struct bpf_prog *tgt_prog,
24332			    u32 btf_id,
24333			    struct bpf_attach_target_info *tgt_info)
24334{
24335	bool prog_extension = prog->type == BPF_PROG_TYPE_EXT;
24336	bool prog_tracing = prog->type == BPF_PROG_TYPE_TRACING;
24337	char trace_symbol[KSYM_SYMBOL_LEN];
24338	const char prefix[] = "btf_trace_";
24339	struct bpf_raw_event_map *btp;
24340	int ret = 0, subprog = -1, i;
24341	const struct btf_type *t;
24342	bool conservative = true;
24343	const char *tname, *fname;
24344	struct btf *btf;
24345	long addr = 0;
24346	struct module *mod = NULL;
24347
24348	if (!btf_id) {
24349		bpf_log(log, "Tracing programs must provide btf_id\n");
24350		return -EINVAL;
24351	}
24352	btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf;
24353	if (!btf) {
24354		bpf_log(log,
24355			"FENTRY/FEXIT program can only be attached to another program annotated with BTF\n");
24356		return -EINVAL;
24357	}
24358	t = btf_type_by_id(btf, btf_id);
24359	if (!t) {
24360		bpf_log(log, "attach_btf_id %u is invalid\n", btf_id);
24361		return -EINVAL;
24362	}
24363	tname = btf_name_by_offset(btf, t->name_off);
24364	if (!tname) {
24365		bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id);
24366		return -EINVAL;
24367	}
24368	if (tgt_prog) {
24369		struct bpf_prog_aux *aux = tgt_prog->aux;
24370		bool tgt_changes_pkt_data;
24371		bool tgt_might_sleep;
24372
24373		if (bpf_prog_is_dev_bound(prog->aux) &&
24374		    !bpf_prog_dev_bound_match(prog, tgt_prog)) {
24375			bpf_log(log, "Target program bound device mismatch");
24376			return -EINVAL;
24377		}
24378
24379		for (i = 0; i < aux->func_info_cnt; i++)
24380			if (aux->func_info[i].type_id == btf_id) {
24381				subprog = i;
24382				break;
24383			}
24384		if (subprog == -1) {
24385			bpf_log(log, "Subprog %s doesn't exist\n", tname);
24386			return -EINVAL;
24387		}
24388		if (aux->func && aux->func[subprog]->aux->exception_cb) {
24389			bpf_log(log,
24390				"%s programs cannot attach to exception callback\n",
24391				prog_extension ? "Extension" : "FENTRY/FEXIT");
24392			return -EINVAL;
24393		}
24394		conservative = aux->func_info_aux[subprog].unreliable;
24395		if (prog_extension) {
24396			if (conservative) {
24397				bpf_log(log,
24398					"Cannot replace static functions\n");
24399				return -EINVAL;
24400			}
24401			if (!prog->jit_requested) {
24402				bpf_log(log,
24403					"Extension programs should be JITed\n");
24404				return -EINVAL;
24405			}
24406			tgt_changes_pkt_data = aux->func
24407					       ? aux->func[subprog]->aux->changes_pkt_data
24408					       : aux->changes_pkt_data;
24409			if (prog->aux->changes_pkt_data && !tgt_changes_pkt_data) {
24410				bpf_log(log,
24411					"Extension program changes packet data, while original does not\n");
24412				return -EINVAL;
24413			}
24414
24415			tgt_might_sleep = aux->func
24416					  ? aux->func[subprog]->aux->might_sleep
24417					  : aux->might_sleep;
24418			if (prog->aux->might_sleep && !tgt_might_sleep) {
24419				bpf_log(log,
24420					"Extension program may sleep, while original does not\n");
24421				return -EINVAL;
24422			}
24423		}
24424		if (!tgt_prog->jited) {
24425			bpf_log(log, "Can attach to only JITed progs\n");
24426			return -EINVAL;
24427		}
24428		if (prog_tracing) {
24429			if (aux->attach_tracing_prog) {
24430				/*
24431				 * Target program is an fentry/fexit which is already attached
24432				 * to another tracing program. More levels of nesting
24433				 * attachment are not allowed.
24434				 */
24435				bpf_log(log, "Cannot nest tracing program attach more than once\n");
24436				return -EINVAL;
24437			}
24438		} else if (tgt_prog->type == prog->type) {
24439			/*
24440			 * To avoid potential call chain cycles, prevent attaching of a
24441			 * program extension to another extension. It's ok to attach
24442			 * fentry/fexit to extension program.
24443			 */
24444			bpf_log(log, "Cannot recursively attach\n");
24445			return -EINVAL;
24446		}
24447		if (tgt_prog->type == BPF_PROG_TYPE_TRACING &&
24448		    prog_extension &&
24449		    (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY ||
24450		     tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) {
24451			/* Program extensions can extend all program types
24452			 * except fentry/fexit. The reason is the following.
24453			 * The fentry/fexit programs are used for performance
24454			 * analysis, stats and can be attached to any program
24455			 * type. When extension program is replacing XDP function
24456			 * it is necessary to allow performance analysis of all
24457			 * functions. Both original XDP program and its program
24458			 * extension. Hence attaching fentry/fexit to
24459			 * BPF_PROG_TYPE_EXT is allowed. If extending of
24460			 * fentry/fexit was allowed it would be possible to create
24461			 * long call chain fentry->extension->fentry->extension
24462			 * beyond reasonable stack size. Hence extending fentry
24463			 * is not allowed.
24464			 */
24465			bpf_log(log, "Cannot extend fentry/fexit\n");
24466			return -EINVAL;
24467		}
24468	} else {
24469		if (prog_extension) {
24470			bpf_log(log, "Cannot replace kernel functions\n");
24471			return -EINVAL;
24472		}
24473	}
24474
24475	switch (prog->expected_attach_type) {
24476	case BPF_TRACE_RAW_TP:
24477		if (tgt_prog) {
24478			bpf_log(log,
24479				"Only FENTRY/FEXIT progs are attachable to another BPF prog\n");
24480			return -EINVAL;
24481		}
24482		if (!btf_type_is_typedef(t)) {
24483			bpf_log(log, "attach_btf_id %u is not a typedef\n",
24484				btf_id);
24485			return -EINVAL;
24486		}
24487		if (strncmp(prefix, tname, sizeof(prefix) - 1)) {
24488			bpf_log(log, "attach_btf_id %u points to wrong type name %s\n",
24489				btf_id, tname);
24490			return -EINVAL;
24491		}
24492		tname += sizeof(prefix) - 1;
24493
24494		/* The func_proto of "btf_trace_##tname" is generated from typedef without argument
24495		 * names. Thus using bpf_raw_event_map to get argument names.
24496		 */
24497		btp = bpf_get_raw_tracepoint(tname);
24498		if (!btp)
24499			return -EINVAL;
24500		fname = kallsyms_lookup((unsigned long)btp->bpf_func, NULL, NULL, NULL,
24501					trace_symbol);
24502		bpf_put_raw_tracepoint(btp);
24503
24504		if (fname)
24505			ret = btf_find_by_name_kind(btf, fname, BTF_KIND_FUNC);
24506
24507		if (!fname || ret < 0) {
24508			bpf_log(log, "Cannot find btf of tracepoint template, fall back to %s%s.\n",
24509				prefix, tname);
24510			t = btf_type_by_id(btf, t->type);
24511			if (!btf_type_is_ptr(t))
24512				/* should never happen in valid vmlinux build */
24513				return -EINVAL;
24514		} else {
24515			t = btf_type_by_id(btf, ret);
24516			if (!btf_type_is_func(t))
24517				/* should never happen in valid vmlinux build */
24518				return -EINVAL;
24519		}
24520
24521		t = btf_type_by_id(btf, t->type);
24522		if (!btf_type_is_func_proto(t))
24523			/* should never happen in valid vmlinux build */
24524			return -EINVAL;
24525
24526		break;
24527	case BPF_TRACE_ITER:
24528		if (!btf_type_is_func(t)) {
24529			bpf_log(log, "attach_btf_id %u is not a function\n",
24530				btf_id);
24531			return -EINVAL;
24532		}
24533		t = btf_type_by_id(btf, t->type);
24534		if (!btf_type_is_func_proto(t))
24535			return -EINVAL;
24536		ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
24537		if (ret)
24538			return ret;
24539		break;
24540	default:
24541		if (!prog_extension)
24542			return -EINVAL;
24543		fallthrough;
24544	case BPF_MODIFY_RETURN:
24545	case BPF_LSM_MAC:
24546	case BPF_LSM_CGROUP:
24547	case BPF_TRACE_FENTRY:
24548	case BPF_TRACE_FEXIT:
24549		if (!btf_type_is_func(t)) {
24550			bpf_log(log, "attach_btf_id %u is not a function\n",
24551				btf_id);
24552			return -EINVAL;
24553		}
24554		if (prog_extension &&
24555		    btf_check_type_match(log, prog, btf, t))
24556			return -EINVAL;
24557		t = btf_type_by_id(btf, t->type);
24558		if (!btf_type_is_func_proto(t))
24559			return -EINVAL;
24560
24561		if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) &&
24562		    (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type ||
24563		     prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type))
24564			return -EINVAL;
24565
24566		if (tgt_prog && conservative)
24567			t = NULL;
24568
24569		ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
24570		if (ret < 0)
24571			return ret;
24572
24573		if (tgt_prog) {
24574			if (subprog == 0)
24575				addr = (long) tgt_prog->bpf_func;
24576			else
24577				addr = (long) tgt_prog->aux->func[subprog]->bpf_func;
24578		} else {
24579			if (btf_is_module(btf)) {
24580				mod = btf_try_get_module(btf);
24581				if (mod)
24582					addr = find_kallsyms_symbol_value(mod, tname);
24583				else
24584					addr = 0;
24585			} else {
24586				addr = kallsyms_lookup_name(tname);
24587			}
24588			if (!addr) {
24589				module_put(mod);
24590				bpf_log(log,
24591					"The address of function %s cannot be found\n",
24592					tname);
24593				return -ENOENT;
24594			}
24595		}
24596
24597		if (prog->sleepable) {
24598			ret = -EINVAL;
24599			switch (prog->type) {
24600			case BPF_PROG_TYPE_TRACING:
24601
24602				/* fentry/fexit/fmod_ret progs can be sleepable if they are
24603				 * attached to ALLOW_ERROR_INJECTION and are not in denylist.
24604				 */
24605				if (!check_non_sleepable_error_inject(btf_id) &&
24606				    within_error_injection_list(addr))
24607					ret = 0;
24608				/* fentry/fexit/fmod_ret progs can also be sleepable if they are
24609				 * in the fmodret id set with the KF_SLEEPABLE flag.
24610				 */
24611				else {
24612					u32 *flags = btf_kfunc_is_modify_return(btf, btf_id,
24613										prog);
24614
24615					if (flags && (*flags & KF_SLEEPABLE))
24616						ret = 0;
24617				}
24618				break;
24619			case BPF_PROG_TYPE_LSM:
24620				/* LSM progs check that they are attached to bpf_lsm_*() funcs.
24621				 * Only some of them are sleepable.
24622				 */
24623				if (bpf_lsm_is_sleepable_hook(btf_id))
24624					ret = 0;
24625				break;
24626			default:
24627				break;
24628			}
24629			if (ret) {
24630				module_put(mod);
24631				bpf_log(log, "%s is not sleepable\n", tname);
24632				return ret;
24633			}
24634		} else if (prog->expected_attach_type == BPF_MODIFY_RETURN) {
24635			if (tgt_prog) {
24636				module_put(mod);
24637				bpf_log(log, "can't modify return codes of BPF programs\n");
24638				return -EINVAL;
24639			}
24640			ret = -EINVAL;
24641			if (btf_kfunc_is_modify_return(btf, btf_id, prog) ||
24642			    !check_attach_modify_return(addr, tname))
24643				ret = 0;
24644			if (ret) {
24645				module_put(mod);
24646				bpf_log(log, "%s() is not modifiable\n", tname);
24647				return ret;
24648			}
24649		}
24650
24651		break;
24652	}
24653	tgt_info->tgt_addr = addr;
24654	tgt_info->tgt_name = tname;
24655	tgt_info->tgt_type = t;
24656	tgt_info->tgt_mod = mod;
24657	return 0;
24658}
24659
24660BTF_SET_START(btf_id_deny)
24661BTF_ID_UNUSED
24662#ifdef CONFIG_SMP
24663BTF_ID(func, ___migrate_enable)
24664BTF_ID(func, migrate_disable)
24665BTF_ID(func, migrate_enable)
24666#endif
24667#if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU
24668BTF_ID(func, rcu_read_unlock_strict)
24669#endif
24670#if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_TRACE_PREEMPT_TOGGLE)
24671BTF_ID(func, preempt_count_add)
24672BTF_ID(func, preempt_count_sub)
24673#endif
24674#ifdef CONFIG_PREEMPT_RCU
24675BTF_ID(func, __rcu_read_lock)
24676BTF_ID(func, __rcu_read_unlock)
24677#endif
24678BTF_SET_END(btf_id_deny)
24679
24680/* fexit and fmod_ret can't be used to attach to __noreturn functions.
24681 * Currently, we must manually list all __noreturn functions here. Once a more
24682 * robust solution is implemented, this workaround can be removed.
24683 */
24684BTF_SET_START(noreturn_deny)
24685#ifdef CONFIG_IA32_EMULATION
24686BTF_ID(func, __ia32_sys_exit)
24687BTF_ID(func, __ia32_sys_exit_group)
24688#endif
24689#ifdef CONFIG_KUNIT
24690BTF_ID(func, __kunit_abort)
24691BTF_ID(func, kunit_try_catch_throw)
24692#endif
24693#ifdef CONFIG_MODULES
24694BTF_ID(func, __module_put_and_kthread_exit)
24695#endif
24696#ifdef CONFIG_X86_64
24697BTF_ID(func, __x64_sys_exit)
24698BTF_ID(func, __x64_sys_exit_group)
24699#endif
24700BTF_ID(func, do_exit)
24701BTF_ID(func, do_group_exit)
24702BTF_ID(func, kthread_complete_and_exit)
24703BTF_ID(func, make_task_dead)
24704BTF_SET_END(noreturn_deny)
24705
24706static bool can_be_sleepable(struct bpf_prog *prog)
24707{
24708	if (prog->type == BPF_PROG_TYPE_TRACING) {
24709		switch (prog->expected_attach_type) {
24710		case BPF_TRACE_FENTRY:
24711		case BPF_TRACE_FEXIT:
24712		case BPF_MODIFY_RETURN:
24713		case BPF_TRACE_ITER:
24714			return true;
24715		default:
24716			return false;
24717		}
24718	}
24719	return prog->type == BPF_PROG_TYPE_LSM ||
24720	       prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ ||
24721	       prog->type == BPF_PROG_TYPE_STRUCT_OPS;
24722}
24723
24724static int check_attach_btf_id(struct bpf_verifier_env *env)
24725{
24726	struct bpf_prog *prog = env->prog;
24727	struct bpf_prog *tgt_prog = prog->aux->dst_prog;
24728	struct bpf_attach_target_info tgt_info = {};
24729	u32 btf_id = prog->aux->attach_btf_id;
24730	struct bpf_trampoline *tr;
24731	int ret;
24732	u64 key;
24733
24734	if (prog->type == BPF_PROG_TYPE_SYSCALL) {
24735		if (prog->sleepable)
24736			/* attach_btf_id checked to be zero already */
24737			return 0;
24738		verbose(env, "Syscall programs can only be sleepable\n");
24739		return -EINVAL;
24740	}
24741
24742	if (prog->sleepable && !can_be_sleepable(prog)) {
24743		verbose(env, "Only fentry/fexit/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepable\n");
24744		return -EINVAL;
24745	}
24746
24747	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS)
24748		return check_struct_ops_btf_id(env);
24749
24750	if (prog->type != BPF_PROG_TYPE_TRACING &&
24751	    prog->type != BPF_PROG_TYPE_LSM &&
24752	    prog->type != BPF_PROG_TYPE_EXT)
24753		return 0;
24754
24755	ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info);
24756	if (ret)
24757		return ret;
24758
24759	if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) {
24760		/* to make freplace equivalent to their targets, they need to
24761		 * inherit env->ops and expected_attach_type for the rest of the
24762		 * verification
24763		 */
24764		env->ops = bpf_verifier_ops[tgt_prog->type];
24765		prog->expected_attach_type = tgt_prog->expected_attach_type;
24766	}
24767
24768	/* store info about the attachment target that will be used later */
24769	prog->aux->attach_func_proto = tgt_info.tgt_type;
24770	prog->aux->attach_func_name = tgt_info.tgt_name;
24771	prog->aux->mod = tgt_info.tgt_mod;
24772
24773	if (tgt_prog) {
24774		prog->aux->saved_dst_prog_type = tgt_prog->type;
24775		prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type;
24776	}
24777
24778	if (prog->expected_attach_type == BPF_TRACE_RAW_TP) {
24779		prog->aux->attach_btf_trace = true;
24780		return 0;
24781	} else if (prog->expected_attach_type == BPF_TRACE_ITER) {
24782		return bpf_iter_prog_supported(prog);
24783	}
24784
24785	if (prog->type == BPF_PROG_TYPE_LSM) {
24786		ret = bpf_lsm_verify_prog(&env->log, prog);
24787		if (ret < 0)
24788			return ret;
24789	} else if (prog->type == BPF_PROG_TYPE_TRACING &&
24790		   btf_id_set_contains(&btf_id_deny, btf_id)) {
24791		verbose(env, "Attaching tracing programs to function '%s' is rejected.\n",
24792			tgt_info.tgt_name);
24793		return -EINVAL;
24794	} else if ((prog->expected_attach_type == BPF_TRACE_FEXIT ||
24795		   prog->expected_attach_type == BPF_MODIFY_RETURN) &&
24796		   btf_id_set_contains(&noreturn_deny, btf_id)) {
24797		verbose(env, "Attaching fexit/fmod_ret to __noreturn function '%s' is rejected.\n",
24798			tgt_info.tgt_name);
24799		return -EINVAL;
24800	}
24801
24802	key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id);
24803	tr = bpf_trampoline_get(key, &tgt_info);
24804	if (!tr)
24805		return -ENOMEM;
24806
24807	if (tgt_prog && tgt_prog->aux->tail_call_reachable)
24808		tr->flags = BPF_TRAMP_F_TAIL_CALL_CTX;
24809
24810	prog->aux->dst_trampoline = tr;
24811	return 0;
24812}
24813
24814struct btf *bpf_get_btf_vmlinux(void)
24815{
24816	if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {
24817		mutex_lock(&bpf_verifier_lock);
24818		if (!btf_vmlinux)
24819			btf_vmlinux = btf_parse_vmlinux();
24820		mutex_unlock(&bpf_verifier_lock);
24821	}
24822	return btf_vmlinux;
24823}
24824
24825/*
24826 * The add_fd_from_fd_array() is executed only if fd_array_cnt is non-zero. In
24827 * this case expect that every file descriptor in the array is either a map or
24828 * a BTF. Everything else is considered to be trash.
24829 */
24830static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd)
24831{
24832	struct bpf_map *map;
24833	struct btf *btf;
24834	CLASS(fd, f)(fd);
24835	int err;
24836
24837	map = __bpf_map_get(f);
24838	if (!IS_ERR(map)) {
24839		err = __add_used_map(env, map);
24840		if (err < 0)
24841			return err;
24842		return 0;
24843	}
24844
24845	btf = __btf_get_by_fd(f);
24846	if (!IS_ERR(btf)) {
24847		btf_get(btf);
24848		return __add_used_btf(env, btf);
24849	}
24850
24851	verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd);
24852	return PTR_ERR(map);
24853}
24854
24855static int process_fd_array(struct bpf_verifier_env *env, union bpf_attr *attr, bpfptr_t uattr)
24856{
24857	size_t size = sizeof(int);
24858	int ret;
24859	int fd;
24860	u32 i;
24861
24862	env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);
24863
24864	/*
24865	 * The only difference between old (no fd_array_cnt is given) and new
24866	 * APIs is that in the latter case the fd_array is expected to be
24867	 * continuous and is scanned for map fds right away
24868	 */
24869	if (!attr->fd_array_cnt)
24870		return 0;
24871
24872	/* Check for integer overflow */
24873	if (attr->fd_array_cnt >= (U32_MAX / size)) {
24874		verbose(env, "fd_array_cnt is too big (%u)\n", attr->fd_array_cnt);
24875		return -EINVAL;
24876	}
24877
24878	for (i = 0; i < attr->fd_array_cnt; i++) {
24879		if (copy_from_bpfptr_offset(&fd, env->fd_array, i * size, size))
24880			return -EFAULT;
24881
24882		ret = add_fd_from_fd_array(env, fd);
24883		if (ret)
24884			return ret;
24885	}
24886
24887	return 0;
24888}
24889
24890/* Each field is a register bitmask */
24891struct insn_live_regs {
24892	u16 use;	/* registers read by instruction */
24893	u16 def;	/* registers written by instruction */
24894	u16 in;		/* registers that may be alive before instruction */
24895	u16 out;	/* registers that may be alive after instruction */
24896};
24897
24898/* Bitmask with 1s for all caller saved registers */
24899#define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1)
24900
24901/* Compute info->{use,def} fields for the instruction */
24902static void compute_insn_live_regs(struct bpf_verifier_env *env,
24903				   struct bpf_insn *insn,
24904				   struct insn_live_regs *info)
24905{
24906	struct call_summary cs;
24907	u8 class = BPF_CLASS(insn->code);
24908	u8 code = BPF_OP(insn->code);
24909	u8 mode = BPF_MODE(insn->code);
24910	u16 src = BIT(insn->src_reg);
24911	u16 dst = BIT(insn->dst_reg);
24912	u16 r0  = BIT(0);
24913	u16 def = 0;
24914	u16 use = 0xffff;
24915
24916	switch (class) {
24917	case BPF_LD:
24918		switch (mode) {
24919		case BPF_IMM:
24920			if (BPF_SIZE(insn->code) == BPF_DW) {
24921				def = dst;
24922				use = 0;
24923			}
24924			break;
24925		case BPF_LD | BPF_ABS:
24926		case BPF_LD | BPF_IND:
24927			/* stick with defaults */
24928			break;
24929		}
24930		break;
24931	case BPF_LDX:
24932		switch (mode) {
24933		case BPF_MEM:
24934		case BPF_MEMSX:
24935			def = dst;
24936			use = src;
24937			break;
24938		}
24939		break;
24940	case BPF_ST:
24941		switch (mode) {
24942		case BPF_MEM:
24943			def = 0;
24944			use = dst;
24945			break;
24946		}
24947		break;
24948	case BPF_STX:
24949		switch (mode) {
24950		case BPF_MEM:
24951			def = 0;
24952			use = dst | src;
24953			break;
24954		case BPF_ATOMIC:
24955			switch (insn->imm) {
24956			case BPF_CMPXCHG:
24957				use = r0 | dst | src;
24958				def = r0;
24959				break;
24960			case BPF_LOAD_ACQ:
24961				def = dst;
24962				use = src;
24963				break;
24964			case BPF_STORE_REL:
24965				def = 0;
24966				use = dst | src;
24967				break;
24968			default:
24969				use = dst | src;
24970				if (insn->imm & BPF_FETCH)
24971					def = src;
24972				else
24973					def = 0;
24974			}
24975			break;
24976		}
24977		break;
24978	case BPF_ALU:
24979	case BPF_ALU64:
24980		switch (code) {
24981		case BPF_END:
24982			use = dst;
24983			def = dst;
24984			break;
24985		case BPF_MOV:
24986			def = dst;
24987			if (BPF_SRC(insn->code) == BPF_K)
24988				use = 0;
24989			else
24990				use = src;
24991			break;
24992		default:
24993			def = dst;
24994			if (BPF_SRC(insn->code) == BPF_K)
24995				use = dst;
24996			else
24997				use = dst | src;
24998		}
24999		break;
25000	case BPF_JMP:
25001	case BPF_JMP32:
25002		switch (code) {
25003		case BPF_JA:
25004			def = 0;
25005			if (BPF_SRC(insn->code) == BPF_X)
25006				use = dst;
25007			else
25008				use = 0;
25009			break;
25010		case BPF_JCOND:
25011			def = 0;
25012			use = 0;
25013			break;
25014		case BPF_EXIT:
25015			def = 0;
25016			use = r0;
25017			break;
25018		case BPF_CALL:
25019			def = ALL_CALLER_SAVED_REGS;
25020			use = def & ~BIT(BPF_REG_0);
25021			if (get_call_summary(env, insn, &cs))
25022				use = GENMASK(cs.num_params, 1);
25023			break;
25024		default:
25025			def = 0;
25026			if (BPF_SRC(insn->code) == BPF_K)
25027				use = dst;
25028			else
25029				use = dst | src;
25030		}
25031		break;
25032	}
25033
25034	info->def = def;
25035	info->use = use;
25036}
25037
25038/* Compute may-live registers after each instruction in the program.
25039 * The register is live after the instruction I if it is read by some
25040 * instruction S following I during program execution and is not
25041 * overwritten between I and S.
25042 *
25043 * Store result in env->insn_aux_data[i].live_regs.
25044 */
25045static int compute_live_registers(struct bpf_verifier_env *env)
25046{
25047	struct bpf_insn_aux_data *insn_aux = env->insn_aux_data;
25048	struct bpf_insn *insns = env->prog->insnsi;
25049	struct insn_live_regs *state;
25050	int insn_cnt = env->prog->len;
25051	int err = 0, i, j;
25052	bool changed;
25053
25054	/* Use the following algorithm:
25055	 * - define the following:
25056	 *   - I.use : a set of all registers read by instruction I;
25057	 *   - I.def : a set of all registers written by instruction I;
25058	 *   - I.in  : a set of all registers that may be alive before I execution;
25059	 *   - I.out : a set of all registers that may be alive after I execution;
25060	 *   - insn_successors(I): a set of instructions S that might immediately
25061	 *                         follow I for some program execution;
25062	 * - associate separate empty sets 'I.in' and 'I.out' with each instruction;
25063	 * - visit each instruction in a postorder and update
25064	 *   state[i].in, state[i].out as follows:
25065	 *
25066	 *       state[i].out = U [state[s].in for S in insn_successors(i)]
25067	 *       state[i].in  = (state[i].out / state[i].def) U state[i].use
25068	 *
25069	 *   (where U stands for set union, / stands for set difference)
25070	 * - repeat the computation while {in,out} fields changes for
25071	 *   any instruction.
25072	 */
25073	state = kvcalloc(insn_cnt, sizeof(*state), GFP_KERNEL_ACCOUNT);
25074	if (!state) {
25075		err = -ENOMEM;
25076		goto out;
25077	}
25078
25079	for (i = 0; i < insn_cnt; ++i)
25080		compute_insn_live_regs(env, &insns[i], &state[i]);
25081
25082	changed = true;
25083	while (changed) {
25084		changed = false;
25085		for (i = 0; i < env->cfg.cur_postorder; ++i) {
25086			int insn_idx = env->cfg.insn_postorder[i];
25087			struct insn_live_regs *live = &state[insn_idx];
25088			struct bpf_iarray *succ;
25089			u16 new_out = 0;
25090			u16 new_in = 0;
25091
25092			succ = bpf_insn_successors(env, insn_idx);
25093			for (int s = 0; s < succ->cnt; ++s)
25094				new_out |= state[succ->items[s]].in;
25095			new_in = (new_out & ~live->def) | live->use;
25096			if (new_out != live->out || new_in != live->in) {
25097				live->in = new_in;
25098				live->out = new_out;
25099				changed = true;
25100			}
25101		}
25102	}
25103
25104	for (i = 0; i < insn_cnt; ++i)
25105		insn_aux[i].live_regs_before = state[i].in;
25106
25107	if (env->log.level & BPF_LOG_LEVEL2) {
25108		verbose(env, "Live regs before insn:\n");
25109		for (i = 0; i < insn_cnt; ++i) {
25110			if (env->insn_aux_data[i].scc)
25111				verbose(env, "%3d ", env->insn_aux_data[i].scc);
25112			else
25113				verbose(env, "    ");
25114			verbose(env, "%3d: ", i);
25115			for (j = BPF_REG_0; j < BPF_REG_10; ++j)
25116				if (insn_aux[i].live_regs_before & BIT(j))
25117					verbose(env, "%d", j);
25118				else
25119					verbose(env, ".");
25120			verbose(env, " ");
25121			verbose_insn(env, &insns[i]);
25122			if (bpf_is_ldimm64(&insns[i]))
25123				i++;
25124		}
25125	}
25126
25127out:
25128	kvfree(state);
25129	return err;
25130}
25131
25132/*
25133 * Compute strongly connected components (SCCs) on the CFG.
25134 * Assign an SCC number to each instruction, recorded in env->insn_aux[*].scc.
25135 * If instruction is a sole member of its SCC and there are no self edges,
25136 * assign it SCC number of zero.
25137 * Uses a non-recursive adaptation of Tarjan's algorithm for SCC computation.
25138 */
25139static int compute_scc(struct bpf_verifier_env *env)
25140{
25141	const u32 NOT_ON_STACK = U32_MAX;
25142
25143	struct bpf_insn_aux_data *aux = env->insn_aux_data;
25144	const u32 insn_cnt = env->prog->len;
25145	int stack_sz, dfs_sz, err = 0;
25146	u32 *stack, *pre, *low, *dfs;
25147	u32 i, j, t, w;
25148	u32 next_preorder_num;
25149	u32 next_scc_id;
25150	bool assign_scc;
25151	struct bpf_iarray *succ;
25152
25153	next_preorder_num = 1;
25154	next_scc_id = 1;
25155	/*
25156	 * - 'stack' accumulates vertices in DFS order, see invariant comment below;
25157	 * - 'pre[t] == p' => preorder number of vertex 't' is 'p';
25158	 * - 'low[t] == n' => smallest preorder number of the vertex reachable from 't' is 'n';
25159	 * - 'dfs' DFS traversal stack, used to emulate explicit recursion.
25160	 */
25161	stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
25162	pre = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
25163	low = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
25164	dfs = kvcalloc(insn_cnt, sizeof(*dfs), GFP_KERNEL_ACCOUNT);
25165	if (!stack || !pre || !low || !dfs) {
25166		err = -ENOMEM;
25167		goto exit;
25168	}
25169	/*
25170	 * References:
25171	 * [1] R. Tarjan "Depth-First Search and Linear Graph Algorithms"
25172	 * [2] D. J. Pearce "A Space-Efficient Algorithm for Finding Strongly Connected Components"
25173	 *
25174	 * The algorithm maintains the following invariant:
25175	 * - suppose there is a path 'u' ~> 'v', such that 'pre[v] < pre[u]';
25176	 * - then, vertex 'u' remains on stack while vertex 'v' is on stack.
25177	 *
25178	 * Consequently:
25179	 * - If 'low[v] < pre[v]', there is a path from 'v' to some vertex 'u',
25180	 *   such that 'pre[u] == low[v]'; vertex 'u' is currently on the stack,
25181	 *   and thus there is an SCC (loop) containing both 'u' and 'v'.
25182	 * - If 'low[v] == pre[v]', loops containing 'v' have been explored,
25183	 *   and 'v' can be considered the root of some SCC.
25184	 *
25185	 * Here is a pseudo-code for an explicitly recursive version of the algorithm:
25186	 *
25187	 *    NOT_ON_STACK = insn_cnt + 1
25188	 *    pre = [0] * insn_cnt
25189	 *    low = [0] * insn_cnt
25190	 *    scc = [0] * insn_cnt
25191	 *    stack = []
25192	 *
25193	 *    next_preorder_num = 1
25194	 *    next_scc_id = 1
25195	 *
25196	 *    def recur(w):
25197	 *        nonlocal next_preorder_num
25198	 *        nonlocal next_scc_id
25199	 *
25200	 *        pre[w] = next_preorder_num
25201	 *        low[w] = next_preorder_num
25202	 *        next_preorder_num += 1
25203	 *        stack.append(w)
25204	 *        for s in successors(w):
25205	 *            # Note: for classic algorithm the block below should look as:
25206	 *            #
25207	 *            # if pre[s] == 0:
25208	 *            #     recur(s)
25209	 *            #	    low[w] = min(low[w], low[s])
25210	 *            # elif low[s] != NOT_ON_STACK:
25211	 *            #     low[w] = min(low[w], pre[s])
25212	 *            #
25213	 *            # But replacing both 'min' instructions with 'low[w] = min(low[w], low[s])'
25214	 *            # does not break the invariant and makes itartive version of the algorithm
25215	 *            # simpler. See 'Algorithm #3' from [2].
25216	 *
25217	 *            # 's' not yet visited
25218	 *            if pre[s] == 0:
25219	 *                recur(s)
25220	 *            # if 's' is on stack, pick lowest reachable preorder number from it;
25221	 *            # if 's' is not on stack 'low[s] == NOT_ON_STACK > low[w]',
25222	 *            # so 'min' would be a noop.
25223	 *            low[w] = min(low[w], low[s])
25224	 *
25225	 *        if low[w] == pre[w]:
25226	 *            # 'w' is the root of an SCC, pop all vertices
25227	 *            # below 'w' on stack and assign same SCC to them.
25228	 *            while True:
25229	 *                t = stack.pop()
25230	 *                low[t] = NOT_ON_STACK
25231	 *                scc[t] = next_scc_id
25232	 *                if t == w:
25233	 *                    break
25234	 *            next_scc_id += 1
25235	 *
25236	 *    for i in range(0, insn_cnt):
25237	 *        if pre[i] == 0:
25238	 *            recur(i)
25239	 *
25240	 * Below implementation replaces explicit recursion with array 'dfs'.
25241	 */
25242	for (i = 0; i < insn_cnt; i++) {
25243		if (pre[i])
25244			continue;
25245		stack_sz = 0;
25246		dfs_sz = 1;
25247		dfs[0] = i;
25248dfs_continue:
25249		while (dfs_sz) {
25250			w = dfs[dfs_sz - 1];
25251			if (pre[w] == 0) {
25252				low[w] = next_preorder_num;
25253				pre[w] = next_preorder_num;
25254				next_preorder_num++;
25255				stack[stack_sz++] = w;
25256			}
25257			/* Visit 'w' successors */
25258			succ = bpf_insn_successors(env, w);
25259			for (j = 0; j < succ->cnt; ++j) {
25260				if (pre[succ->items[j]]) {
25261					low[w] = min(low[w], low[succ->items[j]]);
25262				} else {
25263					dfs[dfs_sz++] = succ->items[j];
25264					goto dfs_continue;
25265				}
25266			}
25267			/*
25268			 * Preserve the invariant: if some vertex above in the stack
25269			 * is reachable from 'w', keep 'w' on the stack.
25270			 */
25271			if (low[w] < pre[w]) {
25272				dfs_sz--;
25273				goto dfs_continue;
25274			}
25275			/*
25276			 * Assign SCC number only if component has two or more elements,
25277			 * or if component has a self reference, or if instruction is a
25278			 * callback calling function (implicit loop).
25279			 */
25280			assign_scc = stack[stack_sz - 1] != w;	/* two or more elements? */
25281			for (j = 0; j < succ->cnt; ++j) {	/* self reference? */
25282				if (succ->items[j] == w) {
25283					assign_scc = true;
25284					break;
25285				}
25286			}
25287			if (bpf_calls_callback(env, w)) /* implicit loop? */
25288				assign_scc = true;
25289			/* Pop component elements from stack */
25290			do {
25291				t = stack[--stack_sz];
25292				low[t] = NOT_ON_STACK;
25293				if (assign_scc)
25294					aux[t].scc = next_scc_id;
25295			} while (t != w);
25296			if (assign_scc)
25297				next_scc_id++;
25298			dfs_sz--;
25299		}
25300	}
25301	env->scc_info = kvcalloc(next_scc_id, sizeof(*env->scc_info), GFP_KERNEL_ACCOUNT);
25302	if (!env->scc_info) {
25303		err = -ENOMEM;
25304		goto exit;
25305	}
25306	env->scc_cnt = next_scc_id;
25307exit:
25308	kvfree(stack);
25309	kvfree(pre);
25310	kvfree(low);
25311	kvfree(dfs);
25312	return err;
25313}
25314
25315int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size)
25316{
25317	u64 start_time = ktime_get_ns();
25318	struct bpf_verifier_env *env;
25319	int i, len, ret = -EINVAL, err;
25320	u32 log_true_size;
25321	bool is_priv;
25322
25323	BTF_TYPE_EMIT(enum bpf_features);
25324
25325	/* no program is valid */
25326	if (ARRAY_SIZE(bpf_verifier_ops) == 0)
25327		return -EINVAL;
25328
25329	/* 'struct bpf_verifier_env' can be global, but since it's not small,
25330	 * allocate/free it every time bpf_check() is called
25331	 */
25332	env = kvzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL_ACCOUNT);
25333	if (!env)
25334		return -ENOMEM;
25335
25336	env->bt.env = env;
25337
25338	len = (*prog)->len;
25339	env->insn_aux_data =
25340		vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len));
25341	ret = -ENOMEM;
25342	if (!env->insn_aux_data)
25343		goto err_free_env;
25344	for (i = 0; i < len; i++)
25345		env->insn_aux_data[i].orig_idx = i;
25346	env->succ = iarray_realloc(NULL, 2);
25347	if (!env->succ)
25348		goto err_free_env;
25349	env->prog = *prog;
25350	env->ops = bpf_verifier_ops[env->prog->type];
25351
25352	env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token);
25353	env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token);
25354	env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token);
25355	env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token);
25356	env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF);
25357
25358	bpf_get_btf_vmlinux();
25359
25360	/* grab the mutex to protect few globals used by verifier */
25361	if (!is_priv)
25362		mutex_lock(&bpf_verifier_lock);
25363
25364	/* user could have requested verbose verifier output
25365	 * and supplied buffer to store the verification trace
25366	 */
25367	ret = bpf_vlog_init(&env->log, attr->log_level,
25368			    (char __user *) (unsigned long) attr->log_buf,
25369			    attr->log_size);
25370	if (ret)
25371		goto err_unlock;
25372
25373	ret = process_fd_array(env, attr, uattr);
25374	if (ret)
25375		goto skip_full_check;
25376
25377	mark_verifier_state_clean(env);
25378
25379	if (IS_ERR(btf_vmlinux)) {
25380		/* Either gcc or pahole or kernel are broken. */
25381		verbose(env, "in-kernel BTF is malformed\n");
25382		ret = PTR_ERR(btf_vmlinux);
25383		goto skip_full_check;
25384	}
25385
25386	env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);
25387	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
25388		env->strict_alignment = true;
25389	if (attr->prog_flags & BPF_F_ANY_ALIGNMENT)
25390		env->strict_alignment = false;
25391
25392	if (is_priv)
25393		env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;
25394	env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS;
25395
25396	env->explored_states = kvcalloc(state_htab_size(env),
25397				       sizeof(struct list_head),
25398				       GFP_KERNEL_ACCOUNT);
25399	ret = -ENOMEM;
25400	if (!env->explored_states)
25401		goto skip_full_check;
25402
25403	for (i = 0; i < state_htab_size(env); i++)
25404		INIT_LIST_HEAD(&env->explored_states[i]);
25405	INIT_LIST_HEAD(&env->free_list);
25406
25407	ret = check_btf_info_early(env, attr, uattr);
25408	if (ret < 0)
25409		goto skip_full_check;
25410
25411	ret = add_subprog_and_kfunc(env);
25412	if (ret < 0)
25413		goto skip_full_check;
25414
25415	ret = check_subprogs(env);
25416	if (ret < 0)
25417		goto skip_full_check;
25418
25419	ret = check_btf_info(env, attr, uattr);
25420	if (ret < 0)
25421		goto skip_full_check;
25422
25423	ret = resolve_pseudo_ldimm64(env);
25424	if (ret < 0)
25425		goto skip_full_check;
25426
25427	if (bpf_prog_is_offloaded(env->prog->aux)) {
25428		ret = bpf_prog_offload_verifier_prep(env->prog);
25429		if (ret)
25430			goto skip_full_check;
25431	}
25432
25433	ret = check_cfg(env);
25434	if (ret < 0)
25435		goto skip_full_check;
25436
25437	ret = compute_postorder(env);
25438	if (ret < 0)
25439		goto skip_full_check;
25440
25441	ret = bpf_stack_liveness_init(env);
25442	if (ret)
25443		goto skip_full_check;
25444
25445	ret = check_attach_btf_id(env);
25446	if (ret)
25447		goto skip_full_check;
25448
25449	ret = compute_scc(env);
25450	if (ret < 0)
25451		goto skip_full_check;
25452
25453	ret = compute_live_registers(env);
25454	if (ret < 0)
25455		goto skip_full_check;
25456
25457	ret = mark_fastcall_patterns(env);
25458	if (ret < 0)
25459		goto skip_full_check;
25460
25461	ret = do_check_main(env);
25462	ret = ret ?: do_check_subprogs(env);
25463
25464	if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux))
25465		ret = bpf_prog_offload_finalize(env);
25466
25467skip_full_check:
25468	kvfree(env->explored_states);
25469
25470	/* might decrease stack depth, keep it before passes that
25471	 * allocate additional slots.
25472	 */
25473	if (ret == 0)
25474		ret = remove_fastcall_spills_fills(env);
25475
25476	if (ret == 0)
25477		ret = check_max_stack_depth(env);
25478
25479	/* instruction rewrites happen after this point */
25480	if (ret == 0)
25481		ret = optimize_bpf_loop(env);
25482
25483	if (is_priv) {
25484		if (ret == 0)
25485			opt_hard_wire_dead_code_branches(env);
25486		if (ret == 0)
25487			ret = opt_remove_dead_code(env);
25488		if (ret == 0)
25489			ret = opt_remove_nops(env);
25490	} else {
25491		if (ret == 0)
25492			sanitize_dead_code(env);
25493	}
25494
25495	if (ret == 0)
25496		/* program is valid, convert *(u32*)(ctx + off) accesses */
25497		ret = convert_ctx_accesses(env);
25498
25499	if (ret == 0)
25500		ret = do_misc_fixups(env);
25501
25502	/* do 32-bit optimization after insn patching has done so those patched
25503	 * insns could be handled correctly.
25504	 */
25505	if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) {
25506		ret = opt_subreg_zext_lo32_rnd_hi32(env, attr);
25507		env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret
25508								     : false;
25509	}
25510
25511	if (ret == 0)
25512		ret = fixup_call_args(env);
25513
25514	env->verification_time = ktime_get_ns() - start_time;
25515	print_verification_stats(env);
25516	env->prog->aux->verified_insns = env->insn_processed;
25517
25518	/* preserve original error even if log finalization is successful */
25519	err = bpf_vlog_finalize(&env->log, &log_true_size);
25520	if (err)
25521		ret = err;
25522
25523	if (uattr_size >= offsetofend(union bpf_attr, log_true_size) &&
25524	    copy_to_bpfptr_offset(uattr, offsetof(union bpf_attr, log_true_size),
25525				  &log_true_size, sizeof(log_true_size))) {
25526		ret = -EFAULT;
25527		goto err_release_maps;
25528	}
25529
25530	if (ret)
25531		goto err_release_maps;
25532
25533	if (env->used_map_cnt) {
25534		/* if program passed verifier, update used_maps in bpf_prog_info */
25535		env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
25536							  sizeof(env->used_maps[0]),
25537							  GFP_KERNEL_ACCOUNT);
25538
25539		if (!env->prog->aux->used_maps) {
25540			ret = -ENOMEM;
25541			goto err_release_maps;
25542		}
25543
25544		memcpy(env->prog->aux->used_maps, env->used_maps,
25545		       sizeof(env->used_maps[0]) * env->used_map_cnt);
25546		env->prog->aux->used_map_cnt = env->used_map_cnt;
25547	}
25548	if (env->used_btf_cnt) {
25549		/* if program passed verifier, update used_btfs in bpf_prog_aux */
25550		env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt,
25551							  sizeof(env->used_btfs[0]),
25552							  GFP_KERNEL_ACCOUNT);
25553		if (!env->prog->aux->used_btfs) {
25554			ret = -ENOMEM;
25555			goto err_release_maps;
25556		}
25557
25558		memcpy(env->prog->aux->used_btfs, env->used_btfs,
25559		       sizeof(env->used_btfs[0]) * env->used_btf_cnt);
25560		env->prog->aux->used_btf_cnt = env->used_btf_cnt;
25561	}
25562	if (env->used_map_cnt || env->used_btf_cnt) {
25563		/* program is valid. Convert pseudo bpf_ld_imm64 into generic
25564		 * bpf_ld_imm64 instructions
25565		 */
25566		convert_pseudo_ld_imm64(env);
25567	}
25568
25569	adjust_btf_func(env);
25570
25571err_release_maps:
25572	if (ret)
25573		release_insn_arrays(env);
25574	if (!env->prog->aux->used_maps)
25575		/* if we didn't copy map pointers into bpf_prog_info, release
25576		 * them now. Otherwise free_used_maps() will release them.
25577		 */
25578		release_maps(env);
25579	if (!env->prog->aux->used_btfs)
25580		release_btfs(env);
25581
25582	/* extension progs temporarily inherit the attach_type of their targets
25583	   for verification purposes, so set it back to zero before returning
25584	 */
25585	if (env->prog->type == BPF_PROG_TYPE_EXT)
25586		env->prog->expected_attach_type = 0;
25587
25588	*prog = env->prog;
25589
25590	module_put(env->attach_btf_mod);
25591err_unlock:
25592	if (!is_priv)
25593		mutex_unlock(&bpf_verifier_lock);
25594	clear_insn_aux_data(env, 0, env->prog->len);
25595	vfree(env->insn_aux_data);
25596err_free_env:
25597	bpf_stack_liveness_free(env);
25598	kvfree(env->cfg.insn_postorder);
25599	kvfree(env->scc_info);
25600	kvfree(env->succ);
25601	kvfree(env->gotox_tmp_buf);
25602	kvfree(env);
25603	return ret;
25604}