core.c

    1// SPDX-License-Identifier: GPL-2.0
    2/*
    3 * Performance events core code:
    4 *
    5 *  Copyright (C) 2008 Linutronix GmbH, Thomas Gleixner <tglx@kernel.org>
    6 *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
    7 *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
    8 *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
    9 */
   10
   11#include <linux/fs.h>
   12#include <linux/mm.h>
   13#include <linux/cpu.h>
   14#include <linux/smp.h>
   15#include <linux/idr.h>
   16#include <linux/file.h>
   17#include <linux/poll.h>
   18#include <linux/slab.h>
   19#include <linux/hash.h>
   20#include <linux/tick.h>
   21#include <linux/sysfs.h>
   22#include <linux/dcache.h>
   23#include <linux/percpu.h>
   24#include <linux/ptrace.h>
   25#include <linux/reboot.h>
   26#include <linux/vmstat.h>
   27#include <linux/device.h>
   28#include <linux/export.h>
   29#include <linux/vmalloc.h>
   30#include <linux/hardirq.h>
   31#include <linux/hugetlb.h>
   32#include <linux/rculist.h>
   33#include <linux/uaccess.h>
   34#include <linux/syscalls.h>
   35#include <linux/anon_inodes.h>
   36#include <linux/kernel_stat.h>
   37#include <linux/cgroup.h>
   38#include <linux/perf_event.h>
   39#include <linux/trace_events.h>
   40#include <linux/hw_breakpoint.h>
   41#include <linux/mm_types.h>
   42#include <linux/module.h>
   43#include <linux/mman.h>
   44#include <linux/compat.h>
   45#include <linux/bpf.h>
   46#include <linux/filter.h>
   47#include <linux/namei.h>
   48#include <linux/parser.h>
   49#include <linux/sched/clock.h>
   50#include <linux/sched/mm.h>
   51#include <linux/proc_ns.h>
   52#include <linux/mount.h>
   53#include <linux/min_heap.h>
   54#include <linux/highmem.h>
   55#include <linux/pgtable.h>
   56#include <linux/buildid.h>
   57#include <linux/task_work.h>
   58#include <linux/percpu-rwsem.h>
   59#include <linux/unwind_deferred.h>
   60
   61#include "internal.h"
   62
   63#include <asm/irq_regs.h>
   64
   65typedef int (*remote_function_f)(void *);
   66
   67struct remote_function_call {
   68	struct task_struct	*p;
   69	remote_function_f	func;
   70	void			*info;
   71	int			ret;
   72};
   73
   74static void remote_function(void *data)
   75{
   76	struct remote_function_call *tfc = data;
   77	struct task_struct *p = tfc->p;
   78
   79	if (p) {
   80		/* -EAGAIN */
   81		if (task_cpu(p) != smp_processor_id())
   82			return;
   83
   84		/*
   85		 * Now that we're on right CPU with IRQs disabled, we can test
   86		 * if we hit the right task without races.
   87		 */
   88
   89		tfc->ret = -ESRCH; /* No such (running) process */
   90		if (p != current)
   91			return;
   92	}
   93
   94	tfc->ret = tfc->func(tfc->info);
   95}
   96
   97/**
   98 * task_function_call - call a function on the cpu on which a task runs
   99 * @p:		the task to evaluate
  100 * @func:	the function to be called
  101 * @info:	the function call argument
  102 *
  103 * Calls the function @func when the task is currently running. This might
  104 * be on the current CPU, which just calls the function directly.  This will
  105 * retry due to any failures in smp_call_function_single(), such as if the
  106 * task_cpu() goes offline concurrently.
  107 *
  108 * returns @func return value or -ESRCH or -ENXIO when the process isn't running
  109 */
  110static int
  111task_function_call(struct task_struct *p, remote_function_f func, void *info)
  112{
  113	struct remote_function_call data = {
  114		.p	= p,
  115		.func	= func,
  116		.info	= info,
  117		.ret	= -EAGAIN,
  118	};
  119	int ret;
  120
  121	for (;;) {
  122		ret = smp_call_function_single(task_cpu(p), remote_function,
  123					       &data, 1);
  124		if (!ret)
  125			ret = data.ret;
  126
  127		if (ret != -EAGAIN)
  128			break;
  129
  130		cond_resched();
  131	}
  132
  133	return ret;
  134}
  135
  136/**
  137 * cpu_function_call - call a function on the cpu
  138 * @cpu:	target cpu to queue this function
  139 * @func:	the function to be called
  140 * @info:	the function call argument
  141 *
  142 * Calls the function @func on the remote cpu.
  143 *
  144 * returns: @func return value or -ENXIO when the cpu is offline
  145 */
  146static int cpu_function_call(int cpu, remote_function_f func, void *info)
  147{
  148	struct remote_function_call data = {
  149		.p	= NULL,
  150		.func	= func,
  151		.info	= info,
  152		.ret	= -ENXIO, /* No such CPU */
  153	};
  154
  155	smp_call_function_single(cpu, remote_function, &data, 1);
  156
  157	return data.ret;
  158}
  159
  160enum event_type_t {
  161	EVENT_FLEXIBLE	= 0x01,
  162	EVENT_PINNED	= 0x02,
  163	EVENT_TIME	= 0x04,
  164	EVENT_FROZEN	= 0x08,
  165	/* see ctx_resched() for details */
  166	EVENT_CPU	= 0x10,
  167	EVENT_CGROUP	= 0x20,
  168
  169	/* compound helpers */
  170	EVENT_ALL         = EVENT_FLEXIBLE | EVENT_PINNED,
  171	EVENT_TIME_FROZEN = EVENT_TIME | EVENT_FROZEN,
  172};
  173
  174static inline void __perf_ctx_lock(struct perf_event_context *ctx)
  175{
  176	raw_spin_lock(&ctx->lock);
  177	WARN_ON_ONCE(ctx->is_active & EVENT_FROZEN);
  178}
  179
  180static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
  181			  struct perf_event_context *ctx)
  182{
  183	__perf_ctx_lock(&cpuctx->ctx);
  184	if (ctx)
  185		__perf_ctx_lock(ctx);
  186}
  187
  188static inline void __perf_ctx_unlock(struct perf_event_context *ctx)
  189{
  190	/*
  191	 * If ctx_sched_in() didn't again set any ALL flags, clean up
  192	 * after ctx_sched_out() by clearing is_active.
  193	 */
  194	if (ctx->is_active & EVENT_FROZEN) {
  195		if (!(ctx->is_active & EVENT_ALL))
  196			ctx->is_active = 0;
  197		else
  198			ctx->is_active &= ~EVENT_FROZEN;
  199	}
  200	raw_spin_unlock(&ctx->lock);
  201}
  202
  203static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
  204			    struct perf_event_context *ctx)
  205{
  206	if (ctx)
  207		__perf_ctx_unlock(ctx);
  208	__perf_ctx_unlock(&cpuctx->ctx);
  209}
  210
  211typedef struct {
  212	struct perf_cpu_context *cpuctx;
  213	struct perf_event_context *ctx;
  214} class_perf_ctx_lock_t;
  215
  216static inline void class_perf_ctx_lock_destructor(class_perf_ctx_lock_t *_T)
  217{ perf_ctx_unlock(_T->cpuctx, _T->ctx); }
  218
  219static inline class_perf_ctx_lock_t
  220class_perf_ctx_lock_constructor(struct perf_cpu_context *cpuctx,
  221				struct perf_event_context *ctx)
  222{ perf_ctx_lock(cpuctx, ctx); return (class_perf_ctx_lock_t){ cpuctx, ctx }; }
  223
  224#define TASK_TOMBSTONE ((void *)-1L)
  225
  226static bool is_kernel_event(struct perf_event *event)
  227{
  228	return READ_ONCE(event->owner) == TASK_TOMBSTONE;
  229}
  230
  231static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
  232
  233struct perf_event_context *perf_cpu_task_ctx(void)
  234{
  235	lockdep_assert_irqs_disabled();
  236	return this_cpu_ptr(&perf_cpu_context)->task_ctx;
  237}
  238
  239/*
  240 * On task ctx scheduling...
  241 *
  242 * When !ctx->nr_events a task context will not be scheduled. This means
  243 * we can disable the scheduler hooks (for performance) without leaving
  244 * pending task ctx state.
  245 *
  246 * This however results in two special cases:
  247 *
  248 *  - removing the last event from a task ctx; this is relatively straight
  249 *    forward and is done in __perf_remove_from_context.
  250 *
  251 *  - adding the first event to a task ctx; this is tricky because we cannot
  252 *    rely on ctx->is_active and therefore cannot use event_function_call().
  253 *    See perf_install_in_context().
  254 *
  255 * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.
  256 */
  257
  258typedef void (*event_f)(struct perf_event *, struct perf_cpu_context *,
  259			struct perf_event_context *, void *);
  260
  261struct event_function_struct {
  262	struct perf_event *event;
  263	event_f func;
  264	void *data;
  265};
  266
  267static int event_function(void *info)
  268{
  269	struct event_function_struct *efs = info;
  270	struct perf_event *event = efs->event;
  271	struct perf_event_context *ctx = event->ctx;
  272	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
  273	struct perf_event_context *task_ctx = cpuctx->task_ctx;
  274	int ret = 0;
  275
  276	lockdep_assert_irqs_disabled();
  277
  278	perf_ctx_lock(cpuctx, task_ctx);
  279	/*
  280	 * Since we do the IPI call without holding ctx->lock things can have
  281	 * changed, double check we hit the task we set out to hit.
  282	 */
  283	if (ctx->task) {
  284		if (ctx->task != current) {
  285			ret = -ESRCH;
  286			goto unlock;
  287		}
  288
  289		/*
  290		 * We only use event_function_call() on established contexts,
  291		 * and event_function() is only ever called when active (or
  292		 * rather, we'll have bailed in task_function_call() or the
  293		 * above ctx->task != current test), therefore we must have
  294		 * ctx->is_active here.
  295		 */
  296		WARN_ON_ONCE(!ctx->is_active);
  297		/*
  298		 * And since we have ctx->is_active, cpuctx->task_ctx must
  299		 * match.
  300		 */
  301		WARN_ON_ONCE(task_ctx != ctx);
  302	} else {
  303		WARN_ON_ONCE(&cpuctx->ctx != ctx);
  304	}
  305
  306	efs->func(event, cpuctx, ctx, efs->data);
  307unlock:
  308	perf_ctx_unlock(cpuctx, task_ctx);
  309
  310	return ret;
  311}
  312
  313static void event_function_call(struct perf_event *event, event_f func, void *data)
  314{
  315	struct perf_event_context *ctx = event->ctx;
  316	struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */
  317	struct perf_cpu_context *cpuctx;
  318	struct event_function_struct efs = {
  319		.event = event,
  320		.func = func,
  321		.data = data,
  322	};
  323
  324	if (!event->parent) {
  325		/*
  326		 * If this is a !child event, we must hold ctx::mutex to
  327		 * stabilize the event->ctx relation. See
  328		 * perf_event_ctx_lock().
  329		 */
  330		lockdep_assert_held(&ctx->mutex);
  331	}
  332
  333	if (!task) {
  334		cpu_function_call(event->cpu, event_function, &efs);
  335		return;
  336	}
  337
  338	if (task == TASK_TOMBSTONE)
  339		return;
  340
  341again:
  342	if (!task_function_call(task, event_function, &efs))
  343		return;
  344
  345	local_irq_disable();
  346	cpuctx = this_cpu_ptr(&perf_cpu_context);
  347	perf_ctx_lock(cpuctx, ctx);
  348	/*
  349	 * Reload the task pointer, it might have been changed by
  350	 * a concurrent perf_event_context_sched_out().
  351	 */
  352	task = ctx->task;
  353	if (task == TASK_TOMBSTONE)
  354		goto unlock;
  355	if (ctx->is_active) {
  356		perf_ctx_unlock(cpuctx, ctx);
  357		local_irq_enable();
  358		goto again;
  359	}
  360	func(event, NULL, ctx, data);
  361unlock:
  362	perf_ctx_unlock(cpuctx, ctx);
  363	local_irq_enable();
  364}
  365
  366/*
  367 * Similar to event_function_call() + event_function(), but hard assumes IRQs
  368 * are already disabled and we're on the right CPU.
  369 */
  370static void event_function_local(struct perf_event *event, event_f func, void *data)
  371{
  372	struct perf_event_context *ctx = event->ctx;
  373	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
  374	struct task_struct *task = READ_ONCE(ctx->task);
  375	struct perf_event_context *task_ctx = NULL;
  376
  377	lockdep_assert_irqs_disabled();
  378
  379	if (task) {
  380		if (task == TASK_TOMBSTONE)
  381			return;
  382
  383		task_ctx = ctx;
  384	}
  385
  386	perf_ctx_lock(cpuctx, task_ctx);
  387
  388	task = ctx->task;
  389	if (task == TASK_TOMBSTONE)
  390		goto unlock;
  391
  392	if (task) {
  393		/*
  394		 * We must be either inactive or active and the right task,
  395		 * otherwise we're screwed, since we cannot IPI to somewhere
  396		 * else.
  397		 */
  398		if (ctx->is_active) {
  399			if (WARN_ON_ONCE(task != current))
  400				goto unlock;
  401
  402			if (WARN_ON_ONCE(cpuctx->task_ctx != ctx))
  403				goto unlock;
  404		}
  405	} else {
  406		WARN_ON_ONCE(&cpuctx->ctx != ctx);
  407	}
  408
  409	func(event, cpuctx, ctx, data);
  410unlock:
  411	perf_ctx_unlock(cpuctx, task_ctx);
  412}
  413
  414#define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
  415		       PERF_FLAG_FD_OUTPUT  |\
  416		       PERF_FLAG_PID_CGROUP |\
  417		       PERF_FLAG_FD_CLOEXEC)
  418
  419/*
  420 * branch priv levels that need permission checks
  421 */
  422#define PERF_SAMPLE_BRANCH_PERM_PLM \
  423	(PERF_SAMPLE_BRANCH_KERNEL |\
  424	 PERF_SAMPLE_BRANCH_HV)
  425
  426/*
  427 * perf_sched_events : >0 events exist
  428 */
  429
  430static void perf_sched_delayed(struct work_struct *work);
  431DEFINE_STATIC_KEY_FALSE(perf_sched_events);
  432static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed);
  433static DEFINE_MUTEX(perf_sched_mutex);
  434static atomic_t perf_sched_count;
  435
  436static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events);
  437
  438static atomic_t nr_mmap_events __read_mostly;
  439static atomic_t nr_comm_events __read_mostly;
  440static atomic_t nr_namespaces_events __read_mostly;
  441static atomic_t nr_task_events __read_mostly;
  442static atomic_t nr_freq_events __read_mostly;
  443static atomic_t nr_switch_events __read_mostly;
  444static atomic_t nr_ksymbol_events __read_mostly;
  445static atomic_t nr_bpf_events __read_mostly;
  446static atomic_t nr_cgroup_events __read_mostly;
  447static atomic_t nr_text_poke_events __read_mostly;
  448static atomic_t nr_build_id_events __read_mostly;
  449
  450static LIST_HEAD(pmus);
  451static DEFINE_MUTEX(pmus_lock);
  452static struct srcu_struct pmus_srcu;
  453static cpumask_var_t perf_online_mask;
  454static cpumask_var_t perf_online_core_mask;
  455static cpumask_var_t perf_online_die_mask;
  456static cpumask_var_t perf_online_cluster_mask;
  457static cpumask_var_t perf_online_pkg_mask;
  458static cpumask_var_t perf_online_sys_mask;
  459static struct kmem_cache *perf_event_cache;
  460
  461/*
  462 * perf event paranoia level:
  463 *  -1 - not paranoid at all
  464 *   0 - disallow raw tracepoint access for unpriv
  465 *   1 - disallow cpu events for unpriv
  466 *   2 - disallow kernel profiling for unpriv
  467 */
  468int sysctl_perf_event_paranoid __read_mostly = 2;
  469
  470/* Minimum for 512 kiB + 1 user control page. 'free' kiB per user. */
  471static int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024);
  472
  473/*
  474 * max perf event sample rate
  475 */
  476#define DEFAULT_MAX_SAMPLE_RATE		100000
  477#define DEFAULT_SAMPLE_PERIOD_NS	(NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
  478#define DEFAULT_CPU_TIME_MAX_PERCENT	25
  479
  480int sysctl_perf_event_sample_rate __read_mostly	= DEFAULT_MAX_SAMPLE_RATE;
  481static int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
  482
  483static int max_samples_per_tick __read_mostly	= DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
  484static int perf_sample_period_ns __read_mostly	= DEFAULT_SAMPLE_PERIOD_NS;
  485
  486static int perf_sample_allowed_ns __read_mostly =
  487	DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100;
  488
  489static void update_perf_cpu_limits(void)
  490{
  491	u64 tmp = perf_sample_period_ns;
  492
  493	tmp *= sysctl_perf_cpu_time_max_percent;
  494	tmp = div_u64(tmp, 100);
  495	if (!tmp)
  496		tmp = 1;
  497
  498	WRITE_ONCE(perf_sample_allowed_ns, tmp);
  499}
  500
  501static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc);
  502
  503static int perf_event_max_sample_rate_handler(const struct ctl_table *table, int write,
  504				       void *buffer, size_t *lenp, loff_t *ppos)
  505{
  506	int ret;
  507	int perf_cpu = sysctl_perf_cpu_time_max_percent;
  508	/*
  509	 * If throttling is disabled don't allow the write:
  510	 */
  511	if (write && (perf_cpu == 100 || perf_cpu == 0))
  512		return -EINVAL;
  513
  514	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
  515	if (ret || !write)
  516		return ret;
  517
  518	max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
  519	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
  520	update_perf_cpu_limits();
  521
  522	return 0;
  523}
  524
  525static int perf_cpu_time_max_percent_handler(const struct ctl_table *table, int write,
  526		void *buffer, size_t *lenp, loff_t *ppos)
  527{
  528	int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
  529
  530	if (ret || !write)
  531		return ret;
  532
  533	if (sysctl_perf_cpu_time_max_percent == 100 ||
  534	    sysctl_perf_cpu_time_max_percent == 0) {
  535		printk(KERN_WARNING
  536		       "perf: Dynamic interrupt throttling disabled, can hang your system!\n");
  537		WRITE_ONCE(perf_sample_allowed_ns, 0);
  538	} else {
  539		update_perf_cpu_limits();
  540	}
  541
  542	return 0;
  543}
  544
  545static const struct ctl_table events_core_sysctl_table[] = {
  546	/*
  547	 * User-space relies on this file as a feature check for
  548	 * perf_events being enabled. It's an ABI, do not remove!
  549	 */
  550	{
  551		.procname	= "perf_event_paranoid",
  552		.data		= &sysctl_perf_event_paranoid,
  553		.maxlen		= sizeof(sysctl_perf_event_paranoid),
  554		.mode		= 0644,
  555		.proc_handler	= proc_dointvec,
  556	},
  557	{
  558		.procname	= "perf_event_mlock_kb",
  559		.data		= &sysctl_perf_event_mlock,
  560		.maxlen		= sizeof(sysctl_perf_event_mlock),
  561		.mode		= 0644,
  562		.proc_handler	= proc_dointvec,
  563	},
  564	{
  565		.procname	= "perf_event_max_sample_rate",
  566		.data		= &sysctl_perf_event_sample_rate,
  567		.maxlen		= sizeof(sysctl_perf_event_sample_rate),
  568		.mode		= 0644,
  569		.proc_handler	= perf_event_max_sample_rate_handler,
  570		.extra1		= SYSCTL_ONE,
  571	},
  572	{
  573		.procname	= "perf_cpu_time_max_percent",
  574		.data		= &sysctl_perf_cpu_time_max_percent,
  575		.maxlen		= sizeof(sysctl_perf_cpu_time_max_percent),
  576		.mode		= 0644,
  577		.proc_handler	= perf_cpu_time_max_percent_handler,
  578		.extra1		= SYSCTL_ZERO,
  579		.extra2		= SYSCTL_ONE_HUNDRED,
  580	},
  581};
  582
  583static int __init init_events_core_sysctls(void)
  584{
  585	register_sysctl_init("kernel", events_core_sysctl_table);
  586	return 0;
  587}
  588core_initcall(init_events_core_sysctls);
  589
  590
  591/*
  592 * perf samples are done in some very critical code paths (NMIs).
  593 * If they take too much CPU time, the system can lock up and not
  594 * get any real work done.  This will drop the sample rate when
  595 * we detect that events are taking too long.
  596 */
  597#define NR_ACCUMULATED_SAMPLES 128
  598static DEFINE_PER_CPU(u64, running_sample_length);
  599
  600static u64 __report_avg;
  601static u64 __report_allowed;
  602
  603static void perf_duration_warn(struct irq_work *w)
  604{
  605	printk_ratelimited(KERN_INFO
  606		"perf: interrupt took too long (%lld > %lld), lowering "
  607		"kernel.perf_event_max_sample_rate to %d\n",
  608		__report_avg, __report_allowed,
  609		sysctl_perf_event_sample_rate);
  610}
  611
  612static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn);
  613
  614void perf_sample_event_took(u64 sample_len_ns)
  615{
  616	u64 max_len = READ_ONCE(perf_sample_allowed_ns);
  617	u64 running_len;
  618	u64 avg_len;
  619	u32 max;
  620
  621	if (max_len == 0)
  622		return;
  623
  624	/* Decay the counter by 1 average sample. */
  625	running_len = __this_cpu_read(running_sample_length);
  626	running_len -= running_len/NR_ACCUMULATED_SAMPLES;
  627	running_len += sample_len_ns;
  628	__this_cpu_write(running_sample_length, running_len);
  629
  630	/*
  631	 * Note: this will be biased artificially low until we have
  632	 * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us
  633	 * from having to maintain a count.
  634	 */
  635	avg_len = running_len/NR_ACCUMULATED_SAMPLES;
  636	if (avg_len <= max_len)
  637		return;
  638
  639	__report_avg = avg_len;
  640	__report_allowed = max_len;
  641
  642	/*
  643	 * Compute a throttle threshold 25% below the current duration.
  644	 */
  645	avg_len += avg_len / 4;
  646	max = (TICK_NSEC / 100) * sysctl_perf_cpu_time_max_percent;
  647	if (avg_len < max)
  648		max /= (u32)avg_len;
  649	else
  650		max = 1;
  651
  652	WRITE_ONCE(perf_sample_allowed_ns, avg_len);
  653	WRITE_ONCE(max_samples_per_tick, max);
  654
  655	sysctl_perf_event_sample_rate = max * HZ;
  656	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
  657
  658	if (!irq_work_queue(&perf_duration_work)) {
  659		early_printk("perf: interrupt took too long (%lld > %lld), lowering "
  660			     "kernel.perf_event_max_sample_rate to %d\n",
  661			     __report_avg, __report_allowed,
  662			     sysctl_perf_event_sample_rate);
  663	}
  664}
  665
  666static atomic64_t perf_event_id;
  667
  668static void update_context_time(struct perf_event_context *ctx);
  669static u64 perf_event_time(struct perf_event *event);
  670
  671void __weak perf_event_print_debug(void)	{ }
  672
  673static inline u64 perf_clock(void)
  674{
  675	return local_clock();
  676}
  677
  678static inline u64 perf_event_clock(struct perf_event *event)
  679{
  680	return event->clock();
  681}
  682
  683/*
  684 * State based event timekeeping...
  685 *
  686 * The basic idea is to use event->state to determine which (if any) time
  687 * fields to increment with the current delta. This means we only need to
  688 * update timestamps when we change state or when they are explicitly requested
  689 * (read).
  690 *
  691 * Event groups make things a little more complicated, but not terribly so. The
  692 * rules for a group are that if the group leader is OFF the entire group is
  693 * OFF, irrespective of what the group member states are. This results in
  694 * __perf_effective_state().
  695 *
  696 * A further ramification is that when a group leader flips between OFF and
  697 * !OFF, we need to update all group member times.
  698 *
  699 *
  700 * NOTE: perf_event_time() is based on the (cgroup) context time, and thus we
  701 * need to make sure the relevant context time is updated before we try and
  702 * update our timestamps.
  703 */
  704
  705static __always_inline enum perf_event_state
  706__perf_effective_state(struct perf_event *event)
  707{
  708	struct perf_event *leader = event->group_leader;
  709
  710	if (leader->state <= PERF_EVENT_STATE_OFF)
  711		return leader->state;
  712
  713	return event->state;
  714}
  715
  716static __always_inline void
  717__perf_update_times(struct perf_event *event, u64 now, u64 *enabled, u64 *running)
  718{
  719	enum perf_event_state state = __perf_effective_state(event);
  720	u64 delta = now - event->tstamp;
  721
  722	*enabled = event->total_time_enabled;
  723	if (state >= PERF_EVENT_STATE_INACTIVE)
  724		*enabled += delta;
  725
  726	*running = event->total_time_running;
  727	if (state >= PERF_EVENT_STATE_ACTIVE)
  728		*running += delta;
  729}
  730
  731static void perf_event_update_time(struct perf_event *event)
  732{
  733	u64 now = perf_event_time(event);
  734
  735	__perf_update_times(event, now, &event->total_time_enabled,
  736					&event->total_time_running);
  737	event->tstamp = now;
  738}
  739
  740static void perf_event_update_sibling_time(struct perf_event *leader)
  741{
  742	struct perf_event *sibling;
  743
  744	for_each_sibling_event(sibling, leader)
  745		perf_event_update_time(sibling);
  746}
  747
  748static void
  749perf_event_set_state(struct perf_event *event, enum perf_event_state state)
  750{
  751	if (event->state == state)
  752		return;
  753
  754	perf_event_update_time(event);
  755	/*
  756	 * If a group leader gets enabled/disabled all its siblings
  757	 * are affected too.
  758	 */
  759	if ((event->state < 0) ^ (state < 0))
  760		perf_event_update_sibling_time(event);
  761
  762	WRITE_ONCE(event->state, state);
  763}
  764
  765/*
  766 * UP store-release, load-acquire
  767 */
  768
  769#define __store_release(ptr, val)					\
  770do {									\
  771	barrier();							\
  772	WRITE_ONCE(*(ptr), (val));					\
  773} while (0)
  774
  775#define __load_acquire(ptr)						\
  776({									\
  777	__unqual_scalar_typeof(*(ptr)) ___p = READ_ONCE(*(ptr));	\
  778	barrier();							\
  779	___p;								\
  780})
  781
  782#define for_each_epc(_epc, _ctx, _pmu, _cgroup)				\
  783	list_for_each_entry(_epc, &((_ctx)->pmu_ctx_list), pmu_ctx_entry) \
  784		if (_cgroup && !_epc->nr_cgroups)			\
  785			continue;					\
  786		else if (_pmu && _epc->pmu != _pmu)			\
  787			continue;					\
  788		else
  789
  790static void perf_ctx_disable(struct perf_event_context *ctx, bool cgroup)
  791{
  792	struct perf_event_pmu_context *pmu_ctx;
  793
  794	for_each_epc(pmu_ctx, ctx, NULL, cgroup)
  795		perf_pmu_disable(pmu_ctx->pmu);
  796}
  797
  798static void perf_ctx_enable(struct perf_event_context *ctx, bool cgroup)
  799{
  800	struct perf_event_pmu_context *pmu_ctx;
  801
  802	for_each_epc(pmu_ctx, ctx, NULL, cgroup)
  803		perf_pmu_enable(pmu_ctx->pmu);
  804}
  805
  806static void ctx_sched_out(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type);
  807static void ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type);
  808
  809#ifdef CONFIG_CGROUP_PERF
  810
  811static inline bool
  812perf_cgroup_match(struct perf_event *event)
  813{
  814	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
  815
  816	/* @event doesn't care about cgroup */
  817	if (!event->cgrp)
  818		return true;
  819
  820	/* wants specific cgroup scope but @cpuctx isn't associated with any */
  821	if (!cpuctx->cgrp)
  822		return false;
  823
  824	/*
  825	 * Cgroup scoping is recursive.  An event enabled for a cgroup is
  826	 * also enabled for all its descendant cgroups.  If @cpuctx's
  827	 * cgroup is a descendant of @event's (the test covers identity
  828	 * case), it's a match.
  829	 */
  830	return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
  831				    event->cgrp->css.cgroup);
  832}
  833
  834static inline void perf_detach_cgroup(struct perf_event *event)
  835{
  836	css_put(&event->cgrp->css);
  837	event->cgrp = NULL;
  838}
  839
  840static inline int is_cgroup_event(struct perf_event *event)
  841{
  842	return event->cgrp != NULL;
  843}
  844
  845static inline u64 perf_cgroup_event_time(struct perf_event *event)
  846{
  847	struct perf_cgroup_info *t;
  848
  849	t = per_cpu_ptr(event->cgrp->info, event->cpu);
  850	return t->time;
  851}
  852
  853static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
  854{
  855	struct perf_cgroup_info *t;
  856
  857	t = per_cpu_ptr(event->cgrp->info, event->cpu);
  858	if (!__load_acquire(&t->active))
  859		return t->time;
  860	now += READ_ONCE(t->timeoffset);
  861	return now;
  862}
  863
  864static inline void __update_cgrp_time(struct perf_cgroup_info *info, u64 now, bool adv)
  865{
  866	if (adv)
  867		info->time += now - info->timestamp;
  868	info->timestamp = now;
  869	/*
  870	 * see update_context_time()
  871	 */
  872	WRITE_ONCE(info->timeoffset, info->time - info->timestamp);
  873}
  874
  875static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, bool final)
  876{
  877	struct perf_cgroup *cgrp = cpuctx->cgrp;
  878	struct cgroup_subsys_state *css;
  879	struct perf_cgroup_info *info;
  880
  881	if (cgrp) {
  882		u64 now = perf_clock();
  883
  884		for (css = &cgrp->css; css; css = css->parent) {
  885			cgrp = container_of(css, struct perf_cgroup, css);
  886			info = this_cpu_ptr(cgrp->info);
  887
  888			__update_cgrp_time(info, now, true);
  889			if (final)
  890				__store_release(&info->active, 0);
  891		}
  892	}
  893}
  894
  895static inline void update_cgrp_time_from_event(struct perf_event *event)
  896{
  897	struct perf_cgroup_info *info;
  898
  899	/*
  900	 * ensure we access cgroup data only when needed and
  901	 * when we know the cgroup is pinned (css_get)
  902	 */
  903	if (!is_cgroup_event(event))
  904		return;
  905
  906	info = this_cpu_ptr(event->cgrp->info);
  907	/*
  908	 * Do not update time when cgroup is not active
  909	 */
  910	if (info->active)
  911		__update_cgrp_time(info, perf_clock(), true);
  912}
  913
  914static inline void
  915perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx)
  916{
  917	struct perf_event_context *ctx = &cpuctx->ctx;
  918	struct perf_cgroup *cgrp = cpuctx->cgrp;
  919	struct perf_cgroup_info *info;
  920	struct cgroup_subsys_state *css;
  921
  922	/*
  923	 * ctx->lock held by caller
  924	 * ensure we do not access cgroup data
  925	 * unless we have the cgroup pinned (css_get)
  926	 */
  927	if (!cgrp)
  928		return;
  929
  930	WARN_ON_ONCE(!ctx->nr_cgroups);
  931
  932	for (css = &cgrp->css; css; css = css->parent) {
  933		cgrp = container_of(css, struct perf_cgroup, css);
  934		info = this_cpu_ptr(cgrp->info);
  935		__update_cgrp_time(info, ctx->timestamp, false);
  936		__store_release(&info->active, 1);
  937	}
  938}
  939
  940/*
  941 * reschedule events based on the cgroup constraint of task.
  942 */
  943static void perf_cgroup_switch(struct task_struct *task)
  944{
  945	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
  946	struct perf_cgroup *cgrp;
  947
  948	/*
  949	 * cpuctx->cgrp is set when the first cgroup event enabled,
  950	 * and is cleared when the last cgroup event disabled.
  951	 */
  952	if (READ_ONCE(cpuctx->cgrp) == NULL)
  953		return;
  954
  955	cgrp = perf_cgroup_from_task(task, NULL);
  956	if (READ_ONCE(cpuctx->cgrp) == cgrp)
  957		return;
  958
  959	guard(perf_ctx_lock)(cpuctx, cpuctx->task_ctx);
  960	/*
  961	 * Re-check, could've raced vs perf_remove_from_context().
  962	 */
  963	if (READ_ONCE(cpuctx->cgrp) == NULL)
  964		return;
  965
  966	WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0);
  967
  968	perf_ctx_disable(&cpuctx->ctx, true);
  969
  970	ctx_sched_out(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP);
  971	/*
  972	 * must not be done before ctxswout due
  973	 * to update_cgrp_time_from_cpuctx() in
  974	 * ctx_sched_out()
  975	 */
  976	cpuctx->cgrp = cgrp;
  977	/*
  978	 * set cgrp before ctxsw in to allow
  979	 * perf_cgroup_set_timestamp() in ctx_sched_in()
  980	 * to not have to pass task around
  981	 */
  982	ctx_sched_in(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP);
  983
  984	perf_ctx_enable(&cpuctx->ctx, true);
  985}
  986
  987static int perf_cgroup_ensure_storage(struct perf_event *event,
  988				struct cgroup_subsys_state *css)
  989{
  990	struct perf_cpu_context *cpuctx;
  991	struct perf_event **storage;
  992	int cpu, heap_size, ret = 0;
  993
  994	/*
  995	 * Allow storage to have sufficient space for an iterator for each
  996	 * possibly nested cgroup plus an iterator for events with no cgroup.
  997	 */
  998	for (heap_size = 1; css; css = css->parent)
  999		heap_size++;
 1000
 1001	for_each_possible_cpu(cpu) {
 1002		cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
 1003		if (heap_size <= cpuctx->heap_size)
 1004			continue;
 1005
 1006		storage = kmalloc_node(heap_size * sizeof(struct perf_event *),
 1007				       GFP_KERNEL, cpu_to_node(cpu));
 1008		if (!storage) {
 1009			ret = -ENOMEM;
 1010			break;
 1011		}
 1012
 1013		raw_spin_lock_irq(&cpuctx->ctx.lock);
 1014		if (cpuctx->heap_size < heap_size) {
 1015			swap(cpuctx->heap, storage);
 1016			if (storage == cpuctx->heap_default)
 1017				storage = NULL;
 1018			cpuctx->heap_size = heap_size;
 1019		}
 1020		raw_spin_unlock_irq(&cpuctx->ctx.lock);
 1021
 1022		kfree(storage);
 1023	}
 1024
 1025	return ret;
 1026}
 1027
 1028static inline int perf_cgroup_connect(int fd, struct perf_event *event,
 1029				      struct perf_event_attr *attr,
 1030				      struct perf_event *group_leader)
 1031{
 1032	struct perf_cgroup *cgrp;
 1033	struct cgroup_subsys_state *css;
 1034	CLASS(fd, f)(fd);
 1035	int ret = 0;
 1036
 1037	if (fd_empty(f))
 1038		return -EBADF;
 1039
 1040	css = css_tryget_online_from_dir(fd_file(f)->f_path.dentry,
 1041					 &perf_event_cgrp_subsys);
 1042	if (IS_ERR(css))
 1043		return PTR_ERR(css);
 1044
 1045	ret = perf_cgroup_ensure_storage(event, css);
 1046	if (ret)
 1047		return ret;
 1048
 1049	cgrp = container_of(css, struct perf_cgroup, css);
 1050	event->cgrp = cgrp;
 1051
 1052	/*
 1053	 * all events in a group must monitor
 1054	 * the same cgroup because a task belongs
 1055	 * to only one perf cgroup at a time
 1056	 */
 1057	if (group_leader && group_leader->cgrp != cgrp) {
 1058		perf_detach_cgroup(event);
 1059		ret = -EINVAL;
 1060	}
 1061	return ret;
 1062}
 1063
 1064static inline void
 1065perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
 1066{
 1067	struct perf_cpu_context *cpuctx;
 1068
 1069	if (!is_cgroup_event(event))
 1070		return;
 1071
 1072	event->pmu_ctx->nr_cgroups++;
 1073
 1074	/*
 1075	 * Because cgroup events are always per-cpu events,
 1076	 * @ctx == &cpuctx->ctx.
 1077	 */
 1078	cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
 1079
 1080	if (ctx->nr_cgroups++)
 1081		return;
 1082
 1083	cpuctx->cgrp = perf_cgroup_from_task(current, ctx);
 1084}
 1085
 1086static inline void
 1087perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)
 1088{
 1089	struct perf_cpu_context *cpuctx;
 1090
 1091	if (!is_cgroup_event(event))
 1092		return;
 1093
 1094	event->pmu_ctx->nr_cgroups--;
 1095
 1096	/*
 1097	 * Because cgroup events are always per-cpu events,
 1098	 * @ctx == &cpuctx->ctx.
 1099	 */
 1100	cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
 1101
 1102	if (--ctx->nr_cgroups)
 1103		return;
 1104
 1105	cpuctx->cgrp = NULL;
 1106}
 1107
 1108#else /* !CONFIG_CGROUP_PERF */
 1109
 1110static inline bool
 1111perf_cgroup_match(struct perf_event *event)
 1112{
 1113	return true;
 1114}
 1115
 1116static inline void perf_detach_cgroup(struct perf_event *event)
 1117{}
 1118
 1119static inline int is_cgroup_event(struct perf_event *event)
 1120{
 1121	return 0;
 1122}
 1123
 1124static inline void update_cgrp_time_from_event(struct perf_event *event)
 1125{
 1126}
 1127
 1128static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx,
 1129						bool final)
 1130{
 1131}
 1132
 1133static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
 1134				      struct perf_event_attr *attr,
 1135				      struct perf_event *group_leader)
 1136{
 1137	return -EINVAL;
 1138}
 1139
 1140static inline void
 1141perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx)
 1142{
 1143}
 1144
 1145static inline u64 perf_cgroup_event_time(struct perf_event *event)
 1146{
 1147	return 0;
 1148}
 1149
 1150static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
 1151{
 1152	return 0;
 1153}
 1154
 1155static inline void
 1156perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
 1157{
 1158}
 1159
 1160static inline void
 1161perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)
 1162{
 1163}
 1164
 1165static void perf_cgroup_switch(struct task_struct *task)
 1166{
 1167}
 1168#endif
 1169
 1170/*
 1171 * set default to be dependent on timer tick just
 1172 * like original code
 1173 */
 1174#define PERF_CPU_HRTIMER (1000 / HZ)
 1175/*
 1176 * function must be called with interrupts disabled
 1177 */
 1178static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr)
 1179{
 1180	struct perf_cpu_pmu_context *cpc;
 1181	bool rotations;
 1182
 1183	lockdep_assert_irqs_disabled();
 1184
 1185	cpc = container_of(hr, struct perf_cpu_pmu_context, hrtimer);
 1186	rotations = perf_rotate_context(cpc);
 1187
 1188	raw_spin_lock(&cpc->hrtimer_lock);
 1189	if (rotations)
 1190		hrtimer_forward_now(hr, cpc->hrtimer_interval);
 1191	else
 1192		cpc->hrtimer_active = 0;
 1193	raw_spin_unlock(&cpc->hrtimer_lock);
 1194
 1195	return rotations ? HRTIMER_RESTART : HRTIMER_NORESTART;
 1196}
 1197
 1198static void __perf_mux_hrtimer_init(struct perf_cpu_pmu_context *cpc, int cpu)
 1199{
 1200	struct hrtimer *timer = &cpc->hrtimer;
 1201	struct pmu *pmu = cpc->epc.pmu;
 1202	u64 interval;
 1203
 1204	/*
 1205	 * check default is sane, if not set then force to
 1206	 * default interval (1/tick)
 1207	 */
 1208	interval = pmu->hrtimer_interval_ms;
 1209	if (interval < 1)
 1210		interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
 1211
 1212	cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);
 1213
 1214	raw_spin_lock_init(&cpc->hrtimer_lock);
 1215	hrtimer_setup(timer, perf_mux_hrtimer_handler, CLOCK_MONOTONIC,
 1216		      HRTIMER_MODE_ABS_PINNED_HARD);
 1217}
 1218
 1219static int perf_mux_hrtimer_restart(struct perf_cpu_pmu_context *cpc)
 1220{
 1221	struct hrtimer *timer = &cpc->hrtimer;
 1222	unsigned long flags;
 1223
 1224	raw_spin_lock_irqsave(&cpc->hrtimer_lock, flags);
 1225	if (!cpc->hrtimer_active) {
 1226		cpc->hrtimer_active = 1;
 1227		hrtimer_forward_now(timer, cpc->hrtimer_interval);
 1228		hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD);
 1229	}
 1230	raw_spin_unlock_irqrestore(&cpc->hrtimer_lock, flags);
 1231
 1232	return 0;
 1233}
 1234
 1235static int perf_mux_hrtimer_restart_ipi(void *arg)
 1236{
 1237	return perf_mux_hrtimer_restart(arg);
 1238}
 1239
 1240static __always_inline struct perf_cpu_pmu_context *this_cpc(struct pmu *pmu)
 1241{
 1242	return *this_cpu_ptr(pmu->cpu_pmu_context);
 1243}
 1244
 1245void perf_pmu_disable(struct pmu *pmu)
 1246{
 1247	int *count = &this_cpc(pmu)->pmu_disable_count;
 1248	if (!(*count)++)
 1249		pmu->pmu_disable(pmu);
 1250}
 1251
 1252void perf_pmu_enable(struct pmu *pmu)
 1253{
 1254	int *count = &this_cpc(pmu)->pmu_disable_count;
 1255	if (!--(*count))
 1256		pmu->pmu_enable(pmu);
 1257}
 1258
 1259static void perf_assert_pmu_disabled(struct pmu *pmu)
 1260{
 1261	int *count = &this_cpc(pmu)->pmu_disable_count;
 1262	WARN_ON_ONCE(*count == 0);
 1263}
 1264
 1265static inline void perf_pmu_read(struct perf_event *event)
 1266{
 1267	if (event->state == PERF_EVENT_STATE_ACTIVE)
 1268		event->pmu->read(event);
 1269}
 1270
 1271static void get_ctx(struct perf_event_context *ctx)
 1272{
 1273	refcount_inc(&ctx->refcount);
 1274}
 1275
 1276static void free_ctx(struct rcu_head *head)
 1277{
 1278	struct perf_event_context *ctx;
 1279
 1280	ctx = container_of(head, struct perf_event_context, rcu_head);
 1281	kfree(ctx);
 1282}
 1283
 1284static void put_ctx(struct perf_event_context *ctx)
 1285{
 1286	if (refcount_dec_and_test(&ctx->refcount)) {
 1287		if (ctx->parent_ctx)
 1288			put_ctx(ctx->parent_ctx);
 1289		if (ctx->task && ctx->task != TASK_TOMBSTONE)
 1290			put_task_struct(ctx->task);
 1291		call_rcu(&ctx->rcu_head, free_ctx);
 1292	} else {
 1293		smp_mb__after_atomic(); /* pairs with wait_var_event() */
 1294		if (ctx->task == TASK_TOMBSTONE)
 1295			wake_up_var(&ctx->refcount);
 1296	}
 1297}
 1298
 1299/*
 1300 * Because of perf_event::ctx migration in sys_perf_event_open::move_group and
 1301 * perf_pmu_migrate_context() we need some magic.
 1302 *
 1303 * Those places that change perf_event::ctx will hold both
 1304 * perf_event_ctx::mutex of the 'old' and 'new' ctx value.
 1305 *
 1306 * Lock ordering is by mutex address. There are two other sites where
 1307 * perf_event_context::mutex nests and those are:
 1308 *
 1309 *  - perf_event_exit_task_context()	[ child , 0 ]
 1310 *      perf_event_exit_event()
 1311 *        put_event()			[ parent, 1 ]
 1312 *
 1313 *  - perf_event_init_context()		[ parent, 0 ]
 1314 *      inherit_task_group()
 1315 *        inherit_group()
 1316 *          inherit_event()
 1317 *            perf_event_alloc()
 1318 *              perf_init_event()
 1319 *                perf_try_init_event()	[ child , 1 ]
 1320 *
 1321 * While it appears there is an obvious deadlock here -- the parent and child
 1322 * nesting levels are inverted between the two. This is in fact safe because
 1323 * life-time rules separate them. That is an exiting task cannot fork, and a
 1324 * spawning task cannot (yet) exit.
 1325 *
 1326 * But remember that these are parent<->child context relations, and
 1327 * migration does not affect children, therefore these two orderings should not
 1328 * interact.
 1329 *
 1330 * The change in perf_event::ctx does not affect children (as claimed above)
 1331 * because the sys_perf_event_open() case will install a new event and break
 1332 * the ctx parent<->child relation, and perf_pmu_migrate_context() is only
 1333 * concerned with cpuctx and that doesn't have children.
 1334 *
 1335 * The places that change perf_event::ctx will issue:
 1336 *
 1337 *   perf_remove_from_context();
 1338 *   synchronize_rcu();
 1339 *   perf_install_in_context();
 1340 *
 1341 * to affect the change. The remove_from_context() + synchronize_rcu() should
 1342 * quiesce the event, after which we can install it in the new location. This
 1343 * means that only external vectors (perf_fops, prctl) can perturb the event
 1344 * while in transit. Therefore all such accessors should also acquire
 1345 * perf_event_context::mutex to serialize against this.
 1346 *
 1347 * However; because event->ctx can change while we're waiting to acquire
 1348 * ctx->mutex we must be careful and use the below perf_event_ctx_lock()
 1349 * function.
 1350 *
 1351 * Lock order:
 1352 *    exec_update_lock
 1353 *	task_struct::perf_event_mutex
 1354 *	  perf_event_context::mutex
 1355 *	    perf_event::child_mutex;
 1356 *	      perf_event_context::lock
 1357 *	    mmap_lock
 1358 *	      perf_event::mmap_mutex
 1359 *	        perf_buffer::aux_mutex
 1360 *	      perf_addr_filters_head::lock
 1361 *
 1362 *    cpu_hotplug_lock
 1363 *      pmus_lock
 1364 *	  cpuctx->mutex / perf_event_context::mutex
 1365 */
 1366static struct perf_event_context *
 1367perf_event_ctx_lock_nested(struct perf_event *event, int nesting)
 1368{
 1369	struct perf_event_context *ctx;
 1370
 1371again:
 1372	rcu_read_lock();
 1373	ctx = READ_ONCE(event->ctx);
 1374	if (!refcount_inc_not_zero(&ctx->refcount)) {
 1375		rcu_read_unlock();
 1376		goto again;
 1377	}
 1378	rcu_read_unlock();
 1379
 1380	mutex_lock_nested(&ctx->mutex, nesting);
 1381	if (event->ctx != ctx) {
 1382		mutex_unlock(&ctx->mutex);
 1383		put_ctx(ctx);
 1384		goto again;
 1385	}
 1386
 1387	return ctx;
 1388}
 1389
 1390static inline struct perf_event_context *
 1391perf_event_ctx_lock(struct perf_event *event)
 1392{
 1393	return perf_event_ctx_lock_nested(event, 0);
 1394}
 1395
 1396static void perf_event_ctx_unlock(struct perf_event *event,
 1397				  struct perf_event_context *ctx)
 1398{
 1399	mutex_unlock(&ctx->mutex);
 1400	put_ctx(ctx);
 1401}
 1402
 1403/*
 1404 * This must be done under the ctx->lock, such as to serialize against
 1405 * context_equiv(), therefore we cannot call put_ctx() since that might end up
 1406 * calling scheduler related locks and ctx->lock nests inside those.
 1407 */
 1408static __must_check struct perf_event_context *
 1409unclone_ctx(struct perf_event_context *ctx)
 1410{
 1411	struct perf_event_context *parent_ctx = ctx->parent_ctx;
 1412
 1413	lockdep_assert_held(&ctx->lock);
 1414
 1415	if (parent_ctx)
 1416		ctx->parent_ctx = NULL;
 1417	ctx->generation++;
 1418
 1419	return parent_ctx;
 1420}
 1421
 1422static u32 perf_event_pid_type(struct perf_event *event, struct task_struct *p,
 1423				enum pid_type type)
 1424{
 1425	u32 nr;
 1426	/*
 1427	 * only top level events have the pid namespace they were created in
 1428	 */
 1429	if (event->parent)
 1430		event = event->parent;
 1431
 1432	nr = __task_pid_nr_ns(p, type, event->ns);
 1433	/* avoid -1 if it is idle thread or runs in another ns */
 1434	if (!nr && !pid_alive(p))
 1435		nr = -1;
 1436	return nr;
 1437}
 1438
 1439static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
 1440{
 1441	return perf_event_pid_type(event, p, PIDTYPE_TGID);
 1442}
 1443
 1444static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
 1445{
 1446	return perf_event_pid_type(event, p, PIDTYPE_PID);
 1447}
 1448
 1449/*
 1450 * If we inherit events we want to return the parent event id
 1451 * to userspace.
 1452 */
 1453static u64 primary_event_id(struct perf_event *event)
 1454{
 1455	u64 id = event->id;
 1456
 1457	if (event->parent)
 1458		id = event->parent->id;
 1459
 1460	return id;
 1461}
 1462
 1463/*
 1464 * Get the perf_event_context for a task and lock it.
 1465 *
 1466 * This has to cope with the fact that until it is locked,
 1467 * the context could get moved to another task.
 1468 */
 1469static struct perf_event_context *
 1470perf_lock_task_context(struct task_struct *task, unsigned long *flags)
 1471{
 1472	struct perf_event_context *ctx;
 1473
 1474retry:
 1475	/*
 1476	 * One of the few rules of preemptible RCU is that one cannot do
 1477	 * rcu_read_unlock() while holding a scheduler (or nested) lock when
 1478	 * part of the read side critical section was irqs-enabled -- see
 1479	 * rcu_read_unlock_special().
 1480	 *
 1481	 * Since ctx->lock nests under rq->lock we must ensure the entire read
 1482	 * side critical section has interrupts disabled.
 1483	 */
 1484	local_irq_save(*flags);
 1485	rcu_read_lock();
 1486	ctx = rcu_dereference(task->perf_event_ctxp);
 1487	if (ctx) {
 1488		/*
 1489		 * If this context is a clone of another, it might
 1490		 * get swapped for another underneath us by
 1491		 * perf_event_task_sched_out, though the
 1492		 * rcu_read_lock() protects us from any context
 1493		 * getting freed.  Lock the context and check if it
 1494		 * got swapped before we could get the lock, and retry
 1495		 * if so.  If we locked the right context, then it
 1496		 * can't get swapped on us any more.
 1497		 */
 1498		raw_spin_lock(&ctx->lock);
 1499		if (ctx != rcu_dereference(task->perf_event_ctxp)) {
 1500			raw_spin_unlock(&ctx->lock);
 1501			rcu_read_unlock();
 1502			local_irq_restore(*flags);
 1503			goto retry;
 1504		}
 1505
 1506		if (ctx->task == TASK_TOMBSTONE ||
 1507		    !refcount_inc_not_zero(&ctx->refcount)) {
 1508			raw_spin_unlock(&ctx->lock);
 1509			ctx = NULL;
 1510		} else {
 1511			WARN_ON_ONCE(ctx->task != task);
 1512		}
 1513	}
 1514	rcu_read_unlock();
 1515	if (!ctx)
 1516		local_irq_restore(*flags);
 1517	return ctx;
 1518}
 1519
 1520/*
 1521 * Get the context for a task and increment its pin_count so it
 1522 * can't get swapped to another task.  This also increments its
 1523 * reference count so that the context can't get freed.
 1524 */
 1525static struct perf_event_context *
 1526perf_pin_task_context(struct task_struct *task)
 1527{
 1528	struct perf_event_context *ctx;
 1529	unsigned long flags;
 1530
 1531	ctx = perf_lock_task_context(task, &flags);
 1532	if (ctx) {
 1533		++ctx->pin_count;
 1534		raw_spin_unlock_irqrestore(&ctx->lock, flags);
 1535	}
 1536	return ctx;
 1537}
 1538
 1539static void perf_unpin_context(struct perf_event_context *ctx)
 1540{
 1541	unsigned long flags;
 1542
 1543	raw_spin_lock_irqsave(&ctx->lock, flags);
 1544	--ctx->pin_count;
 1545	raw_spin_unlock_irqrestore(&ctx->lock, flags);
 1546}
 1547
 1548/*
 1549 * Update the record of the current time in a context.
 1550 */
 1551static void __update_context_time(struct perf_event_context *ctx, bool adv)
 1552{
 1553	u64 now = perf_clock();
 1554
 1555	lockdep_assert_held(&ctx->lock);
 1556
 1557	if (adv)
 1558		ctx->time += now - ctx->timestamp;
 1559	ctx->timestamp = now;
 1560
 1561	/*
 1562	 * The above: time' = time + (now - timestamp), can be re-arranged
 1563	 * into: time` = now + (time - timestamp), which gives a single value
 1564	 * offset to compute future time without locks on.
 1565	 *
 1566	 * See perf_event_time_now(), which can be used from NMI context where
 1567	 * it's (obviously) not possible to acquire ctx->lock in order to read
 1568	 * both the above values in a consistent manner.
 1569	 */
 1570	WRITE_ONCE(ctx->timeoffset, ctx->time - ctx->timestamp);
 1571}
 1572
 1573static void update_context_time(struct perf_event_context *ctx)
 1574{
 1575	__update_context_time(ctx, true);
 1576}
 1577
 1578static u64 perf_event_time(struct perf_event *event)
 1579{
 1580	struct perf_event_context *ctx = event->ctx;
 1581
 1582	if (unlikely(!ctx))
 1583		return 0;
 1584
 1585	if (is_cgroup_event(event))
 1586		return perf_cgroup_event_time(event);
 1587
 1588	return ctx->time;
 1589}
 1590
 1591static u64 perf_event_time_now(struct perf_event *event, u64 now)
 1592{
 1593	struct perf_event_context *ctx = event->ctx;
 1594
 1595	if (unlikely(!ctx))
 1596		return 0;
 1597
 1598	if (is_cgroup_event(event))
 1599		return perf_cgroup_event_time_now(event, now);
 1600
 1601	if (!(__load_acquire(&ctx->is_active) & EVENT_TIME))
 1602		return ctx->time;
 1603
 1604	now += READ_ONCE(ctx->timeoffset);
 1605	return now;
 1606}
 1607
 1608static enum event_type_t get_event_type(struct perf_event *event)
 1609{
 1610	struct perf_event_context *ctx = event->ctx;
 1611	enum event_type_t event_type;
 1612
 1613	lockdep_assert_held(&ctx->lock);
 1614
 1615	/*
 1616	 * It's 'group type', really, because if our group leader is
 1617	 * pinned, so are we.
 1618	 */
 1619	if (event->group_leader != event)
 1620		event = event->group_leader;
 1621
 1622	event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE;
 1623	if (!ctx->task)
 1624		event_type |= EVENT_CPU;
 1625
 1626	return event_type;
 1627}
 1628
 1629/*
 1630 * Helper function to initialize event group nodes.
 1631 */
 1632static void init_event_group(struct perf_event *event)
 1633{
 1634	RB_CLEAR_NODE(&event->group_node);
 1635	event->group_index = 0;
 1636}
 1637
 1638/*
 1639 * Extract pinned or flexible groups from the context
 1640 * based on event attrs bits.
 1641 */
 1642static struct perf_event_groups *
 1643get_event_groups(struct perf_event *event, struct perf_event_context *ctx)
 1644{
 1645	if (event->attr.pinned)
 1646		return &ctx->pinned_groups;
 1647	else
 1648		return &ctx->flexible_groups;
 1649}
 1650
 1651/*
 1652 * Helper function to initializes perf_event_group trees.
 1653 */
 1654static void perf_event_groups_init(struct perf_event_groups *groups)
 1655{
 1656	groups->tree = RB_ROOT;
 1657	groups->index = 0;
 1658}
 1659
 1660static inline struct cgroup *event_cgroup(const struct perf_event *event)
 1661{
 1662	struct cgroup *cgroup = NULL;
 1663
 1664#ifdef CONFIG_CGROUP_PERF
 1665	if (event->cgrp)
 1666		cgroup = event->cgrp->css.cgroup;
 1667#endif
 1668
 1669	return cgroup;
 1670}
 1671
 1672/*
 1673 * Compare function for event groups;
 1674 *
 1675 * Implements complex key that first sorts by CPU and then by virtual index
 1676 * which provides ordering when rotating groups for the same CPU.
 1677 */
 1678static __always_inline int
 1679perf_event_groups_cmp(const int left_cpu, const struct pmu *left_pmu,
 1680		      const struct cgroup *left_cgroup, const u64 left_group_index,
 1681		      const struct perf_event *right)
 1682{
 1683	if (left_cpu < right->cpu)
 1684		return -1;
 1685	if (left_cpu > right->cpu)
 1686		return 1;
 1687
 1688	if (left_pmu) {
 1689		if (left_pmu < right->pmu_ctx->pmu)
 1690			return -1;
 1691		if (left_pmu > right->pmu_ctx->pmu)
 1692			return 1;
 1693	}
 1694
 1695#ifdef CONFIG_CGROUP_PERF
 1696	{
 1697		const struct cgroup *right_cgroup = event_cgroup(right);
 1698
 1699		if (left_cgroup != right_cgroup) {
 1700			if (!left_cgroup) {
 1701				/*
 1702				 * Left has no cgroup but right does, no
 1703				 * cgroups come first.
 1704				 */
 1705				return -1;
 1706			}
 1707			if (!right_cgroup) {
 1708				/*
 1709				 * Right has no cgroup but left does, no
 1710				 * cgroups come first.
 1711				 */
 1712				return 1;
 1713			}
 1714			/* Two dissimilar cgroups, order by id. */
 1715			if (cgroup_id(left_cgroup) < cgroup_id(right_cgroup))
 1716				return -1;
 1717
 1718			return 1;
 1719		}
 1720	}
 1721#endif
 1722
 1723	if (left_group_index < right->group_index)
 1724		return -1;
 1725	if (left_group_index > right->group_index)
 1726		return 1;
 1727
 1728	return 0;
 1729}
 1730
 1731#define __node_2_pe(node) \
 1732	rb_entry((node), struct perf_event, group_node)
 1733
 1734static inline bool __group_less(struct rb_node *a, const struct rb_node *b)
 1735{
 1736	struct perf_event *e = __node_2_pe(a);
 1737	return perf_event_groups_cmp(e->cpu, e->pmu_ctx->pmu, event_cgroup(e),
 1738				     e->group_index, __node_2_pe(b)) < 0;
 1739}
 1740
 1741struct __group_key {
 1742	int cpu;
 1743	struct pmu *pmu;
 1744	struct cgroup *cgroup;
 1745};
 1746
 1747static inline int __group_cmp(const void *key, const struct rb_node *node)
 1748{
 1749	const struct __group_key *a = key;
 1750	const struct perf_event *b = __node_2_pe(node);
 1751
 1752	/* partial/subtree match: @cpu, @pmu, @cgroup; ignore: @group_index */
 1753	return perf_event_groups_cmp(a->cpu, a->pmu, a->cgroup, b->group_index, b);
 1754}
 1755
 1756static inline int
 1757__group_cmp_ignore_cgroup(const void *key, const struct rb_node *node)
 1758{
 1759	const struct __group_key *a = key;
 1760	const struct perf_event *b = __node_2_pe(node);
 1761
 1762	/* partial/subtree match: @cpu, @pmu, ignore: @cgroup, @group_index */
 1763	return perf_event_groups_cmp(a->cpu, a->pmu, event_cgroup(b),
 1764				     b->group_index, b);
 1765}
 1766
 1767/*
 1768 * Insert @event into @groups' tree; using
 1769 *   {@event->cpu, @event->pmu_ctx->pmu, event_cgroup(@event), ++@groups->index}
 1770 * as key. This places it last inside the {cpu,pmu,cgroup} subtree.
 1771 */
 1772static void
 1773perf_event_groups_insert(struct perf_event_groups *groups,
 1774			 struct perf_event *event)
 1775{
 1776	event->group_index = ++groups->index;
 1777
 1778	rb_add(&event->group_node, &groups->tree, __group_less);
 1779}
 1780
 1781/*
 1782 * Helper function to insert event into the pinned or flexible groups.
 1783 */
 1784static void
 1785add_event_to_groups(struct perf_event *event, struct perf_event_context *ctx)
 1786{
 1787	struct perf_event_groups *groups;
 1788
 1789	groups = get_event_groups(event, ctx);
 1790	perf_event_groups_insert(groups, event);
 1791}
 1792
 1793/*
 1794 * Delete a group from a tree.
 1795 */
 1796static void
 1797perf_event_groups_delete(struct perf_event_groups *groups,
 1798			 struct perf_event *event)
 1799{
 1800	WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) ||
 1801		     RB_EMPTY_ROOT(&groups->tree));
 1802
 1803	rb_erase(&event->group_node, &groups->tree);
 1804	init_event_group(event);
 1805}
 1806
 1807/*
 1808 * Helper function to delete event from its groups.
 1809 */
 1810static void
 1811del_event_from_groups(struct perf_event *event, struct perf_event_context *ctx)
 1812{
 1813	struct perf_event_groups *groups;
 1814
 1815	groups = get_event_groups(event, ctx);
 1816	perf_event_groups_delete(groups, event);
 1817}
 1818
 1819/*
 1820 * Get the leftmost event in the {cpu,pmu,cgroup} subtree.
 1821 */
 1822static struct perf_event *
 1823perf_event_groups_first(struct perf_event_groups *groups, int cpu,
 1824			struct pmu *pmu, struct cgroup *cgrp)
 1825{
 1826	struct __group_key key = {
 1827		.cpu = cpu,
 1828		.pmu = pmu,
 1829		.cgroup = cgrp,
 1830	};
 1831	struct rb_node *node;
 1832
 1833	node = rb_find_first(&key, &groups->tree, __group_cmp);
 1834	if (node)
 1835		return __node_2_pe(node);
 1836
 1837	return NULL;
 1838}
 1839
 1840static struct perf_event *
 1841perf_event_groups_next(struct perf_event *event, struct pmu *pmu)
 1842{
 1843	struct __group_key key = {
 1844		.cpu = event->cpu,
 1845		.pmu = pmu,
 1846		.cgroup = event_cgroup(event),
 1847	};
 1848	struct rb_node *next;
 1849
 1850	next = rb_next_match(&key, &event->group_node, __group_cmp);
 1851	if (next)
 1852		return __node_2_pe(next);
 1853
 1854	return NULL;
 1855}
 1856
 1857#define perf_event_groups_for_cpu_pmu(event, groups, cpu, pmu)		\
 1858	for (event = perf_event_groups_first(groups, cpu, pmu, NULL);	\
 1859	     event; event = perf_event_groups_next(event, pmu))
 1860
 1861/*
 1862 * Iterate through the whole groups tree.
 1863 */
 1864#define perf_event_groups_for_each(event, groups)			\
 1865	for (event = rb_entry_safe(rb_first(&((groups)->tree)),		\
 1866				typeof(*event), group_node); event;	\
 1867		event = rb_entry_safe(rb_next(&event->group_node),	\
 1868				typeof(*event), group_node))
 1869
 1870/*
 1871 * Does the event attribute request inherit with PERF_SAMPLE_READ
 1872 */
 1873static inline bool has_inherit_and_sample_read(struct perf_event_attr *attr)
 1874{
 1875	return attr->inherit && (attr->sample_type & PERF_SAMPLE_READ);
 1876}
 1877
 1878/*
 1879 * Add an event from the lists for its context.
 1880 * Must be called with ctx->mutex and ctx->lock held.
 1881 */
 1882static void
 1883list_add_event(struct perf_event *event, struct perf_event_context *ctx)
 1884{
 1885	lockdep_assert_held(&ctx->lock);
 1886
 1887	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
 1888	event->attach_state |= PERF_ATTACH_CONTEXT;
 1889
 1890	event->tstamp = perf_event_time(event);
 1891
 1892	/*
 1893	 * If we're a stand alone event or group leader, we go to the context
 1894	 * list, group events are kept attached to the group so that
 1895	 * perf_group_detach can, at all times, locate all siblings.
 1896	 */
 1897	if (event->group_leader == event) {
 1898		event->group_caps = event->event_caps;
 1899		add_event_to_groups(event, ctx);
 1900	}
 1901
 1902	list_add_rcu(&event->event_entry, &ctx->event_list);
 1903	ctx->nr_events++;
 1904	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)
 1905		ctx->nr_user++;
 1906	if (event->attr.inherit_stat)
 1907		ctx->nr_stat++;
 1908	if (has_inherit_and_sample_read(&event->attr))
 1909		local_inc(&ctx->nr_no_switch_fast);
 1910
 1911	if (event->state > PERF_EVENT_STATE_OFF)
 1912		perf_cgroup_event_enable(event, ctx);
 1913
 1914	ctx->generation++;
 1915	event->pmu_ctx->nr_events++;
 1916}
 1917
 1918/*
 1919 * Initialize event state based on the perf_event_attr::disabled.
 1920 */
 1921static inline void perf_event__state_init(struct perf_event *event)
 1922{
 1923	event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
 1924					      PERF_EVENT_STATE_INACTIVE;
 1925}
 1926
 1927static int __perf_event_read_size(u64 read_format, int nr_siblings)
 1928{
 1929	int entry = sizeof(u64); /* value */
 1930	int size = 0;
 1931	int nr = 1;
 1932
 1933	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
 1934		size += sizeof(u64);
 1935
 1936	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
 1937		size += sizeof(u64);
 1938
 1939	if (read_format & PERF_FORMAT_ID)
 1940		entry += sizeof(u64);
 1941
 1942	if (read_format & PERF_FORMAT_LOST)
 1943		entry += sizeof(u64);
 1944
 1945	if (read_format & PERF_FORMAT_GROUP) {
 1946		nr += nr_siblings;
 1947		size += sizeof(u64);
 1948	}
 1949
 1950	/*
 1951	 * Since perf_event_validate_size() limits this to 16k and inhibits
 1952	 * adding more siblings, this will never overflow.
 1953	 */
 1954	return size + nr * entry;
 1955}
 1956
 1957static void __perf_event_header_size(struct perf_event *event, u64 sample_type)
 1958{
 1959	struct perf_sample_data *data;
 1960	u16 size = 0;
 1961
 1962	if (sample_type & PERF_SAMPLE_IP)
 1963		size += sizeof(data->ip);
 1964
 1965	if (sample_type & PERF_SAMPLE_ADDR)
 1966		size += sizeof(data->addr);
 1967
 1968	if (sample_type & PERF_SAMPLE_PERIOD)
 1969		size += sizeof(data->period);
 1970
 1971	if (sample_type & PERF_SAMPLE_WEIGHT_TYPE)
 1972		size += sizeof(data->weight.full);
 1973
 1974	if (sample_type & PERF_SAMPLE_READ)
 1975		size += event->read_size;
 1976
 1977	if (sample_type & PERF_SAMPLE_DATA_SRC)
 1978		size += sizeof(data->data_src.val);
 1979
 1980	if (sample_type & PERF_SAMPLE_TRANSACTION)
 1981		size += sizeof(data->txn);
 1982
 1983	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
 1984		size += sizeof(data->phys_addr);
 1985
 1986	if (sample_type & PERF_SAMPLE_CGROUP)
 1987		size += sizeof(data->cgroup);
 1988
 1989	if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)
 1990		size += sizeof(data->data_page_size);
 1991
 1992	if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)
 1993		size += sizeof(data->code_page_size);
 1994
 1995	event->header_size = size;
 1996}
 1997
 1998/*
 1999 * Called at perf_event creation and when events are attached/detached from a
 2000 * group.
 2001 */
 2002static void perf_event__header_size(struct perf_event *event)
 2003{
 2004	event->read_size =
 2005		__perf_event_read_size(event->attr.read_format,
 2006				       event->group_leader->nr_siblings);
 2007	__perf_event_header_size(event, event->attr.sample_type);
 2008}
 2009
 2010static void perf_event__id_header_size(struct perf_event *event)
 2011{
 2012	struct perf_sample_data *data;
 2013	u64 sample_type = event->attr.sample_type;
 2014	u16 size = 0;
 2015
 2016	if (sample_type & PERF_SAMPLE_TID)
 2017		size += sizeof(data->tid_entry);
 2018
 2019	if (sample_type & PERF_SAMPLE_TIME)
 2020		size += sizeof(data->time);
 2021
 2022	if (sample_type & PERF_SAMPLE_IDENTIFIER)
 2023		size += sizeof(data->id);
 2024
 2025	if (sample_type & PERF_SAMPLE_ID)
 2026		size += sizeof(data->id);
 2027
 2028	if (sample_type & PERF_SAMPLE_STREAM_ID)
 2029		size += sizeof(data->stream_id);
 2030
 2031	if (sample_type & PERF_SAMPLE_CPU)
 2032		size += sizeof(data->cpu_entry);
 2033
 2034	event->id_header_size = size;
 2035}
 2036
 2037/*
 2038 * Check that adding an event to the group does not result in anybody
 2039 * overflowing the 64k event limit imposed by the output buffer.
 2040 *
 2041 * Specifically, check that the read_size for the event does not exceed 16k,
 2042 * read_size being the one term that grows with groups size. Since read_size
 2043 * depends on per-event read_format, also (re)check the existing events.
 2044 *
 2045 * This leaves 48k for the constant size fields and things like callchains,
 2046 * branch stacks and register sets.
 2047 */
 2048static bool perf_event_validate_size(struct perf_event *event)
 2049{
 2050	struct perf_event *sibling, *group_leader = event->group_leader;
 2051
 2052	if (__perf_event_read_size(event->attr.read_format,
 2053				   group_leader->nr_siblings + 1) > 16*1024)
 2054		return false;
 2055
 2056	if (__perf_event_read_size(group_leader->attr.read_format,
 2057				   group_leader->nr_siblings + 1) > 16*1024)
 2058		return false;
 2059
 2060	/*
 2061	 * When creating a new group leader, group_leader->ctx is initialized
 2062	 * after the size has been validated, but we cannot safely use
 2063	 * for_each_sibling_event() until group_leader->ctx is set. A new group
 2064	 * leader cannot have any siblings yet, so we can safely skip checking
 2065	 * the non-existent siblings.
 2066	 */
 2067	if (event == group_leader)
 2068		return true;
 2069
 2070	for_each_sibling_event(sibling, group_leader) {
 2071		if (__perf_event_read_size(sibling->attr.read_format,
 2072					   group_leader->nr_siblings + 1) > 16*1024)
 2073			return false;
 2074	}
 2075
 2076	return true;
 2077}
 2078
 2079static void perf_group_attach(struct perf_event *event)
 2080{
 2081	struct perf_event *group_leader = event->group_leader, *pos;
 2082
 2083	lockdep_assert_held(&event->ctx->lock);
 2084
 2085	/*
 2086	 * We can have double attach due to group movement (move_group) in
 2087	 * perf_event_open().
 2088	 */
 2089	if (event->attach_state & PERF_ATTACH_GROUP)
 2090		return;
 2091
 2092	event->attach_state |= PERF_ATTACH_GROUP;
 2093
 2094	if (group_leader == event)
 2095		return;
 2096
 2097	WARN_ON_ONCE(group_leader->ctx != event->ctx);
 2098
 2099	group_leader->group_caps &= event->event_caps;
 2100
 2101	list_add_tail(&event->sibling_list, &group_leader->sibling_list);
 2102	group_leader->nr_siblings++;
 2103	group_leader->group_generation++;
 2104
 2105	perf_event__header_size(group_leader);
 2106
 2107	for_each_sibling_event(pos, group_leader)
 2108		perf_event__header_size(pos);
 2109}
 2110
 2111/*
 2112 * Remove an event from the lists for its context.
 2113 * Must be called with ctx->mutex and ctx->lock held.
 2114 */
 2115static void
 2116list_del_event(struct perf_event *event, struct perf_event_context *ctx)
 2117{
 2118	WARN_ON_ONCE(event->ctx != ctx);
 2119	lockdep_assert_held(&ctx->lock);
 2120
 2121	/*
 2122	 * We can have double detach due to exit/hot-unplug + close.
 2123	 */
 2124	if (!(event->attach_state & PERF_ATTACH_CONTEXT))
 2125		return;
 2126
 2127	event->attach_state &= ~PERF_ATTACH_CONTEXT;
 2128
 2129	ctx->nr_events--;
 2130	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)
 2131		ctx->nr_user--;
 2132	if (event->attr.inherit_stat)
 2133		ctx->nr_stat--;
 2134	if (has_inherit_and_sample_read(&event->attr))
 2135		local_dec(&ctx->nr_no_switch_fast);
 2136
 2137	list_del_rcu(&event->event_entry);
 2138
 2139	if (event->group_leader == event)
 2140		del_event_from_groups(event, ctx);
 2141
 2142	ctx->generation++;
 2143	event->pmu_ctx->nr_events--;
 2144}
 2145
 2146static int
 2147perf_aux_output_match(struct perf_event *event, struct perf_event *aux_event)
 2148{
 2149	if (!has_aux(aux_event))
 2150		return 0;
 2151
 2152	if (!event->pmu->aux_output_match)
 2153		return 0;
 2154
 2155	return event->pmu->aux_output_match(aux_event);
 2156}
 2157
 2158static void put_event(struct perf_event *event);
 2159static void __event_disable(struct perf_event *event,
 2160			    struct perf_event_context *ctx,
 2161			    enum perf_event_state state);
 2162
 2163static void perf_put_aux_event(struct perf_event *event)
 2164{
 2165	struct perf_event_context *ctx = event->ctx;
 2166	struct perf_event *iter;
 2167
 2168	/*
 2169	 * If event uses aux_event tear down the link
 2170	 */
 2171	if (event->aux_event) {
 2172		iter = event->aux_event;
 2173		event->aux_event = NULL;
 2174		put_event(iter);
 2175		return;
 2176	}
 2177
 2178	/*
 2179	 * If the event is an aux_event, tear down all links to
 2180	 * it from other events.
 2181	 */
 2182	for_each_sibling_event(iter, event) {
 2183		if (iter->aux_event != event)
 2184			continue;
 2185
 2186		iter->aux_event = NULL;
 2187		put_event(event);
 2188
 2189		/*
 2190		 * If it's ACTIVE, schedule it out and put it into ERROR
 2191		 * state so that we don't try to schedule it again. Note
 2192		 * that perf_event_enable() will clear the ERROR status.
 2193		 */
 2194		__event_disable(iter, ctx, PERF_EVENT_STATE_ERROR);
 2195	}
 2196}
 2197
 2198static bool perf_need_aux_event(struct perf_event *event)
 2199{
 2200	return event->attr.aux_output || has_aux_action(event);
 2201}
 2202
 2203static int perf_get_aux_event(struct perf_event *event,
 2204			      struct perf_event *group_leader)
 2205{
 2206	/*
 2207	 * Our group leader must be an aux event if we want to be
 2208	 * an aux_output. This way, the aux event will precede its
 2209	 * aux_output events in the group, and therefore will always
 2210	 * schedule first.
 2211	 */
 2212	if (!group_leader)
 2213		return 0;
 2214
 2215	/*
 2216	 * aux_output and aux_sample_size are mutually exclusive.
 2217	 */
 2218	if (event->attr.aux_output && event->attr.aux_sample_size)
 2219		return 0;
 2220
 2221	if (event->attr.aux_output &&
 2222	    !perf_aux_output_match(event, group_leader))
 2223		return 0;
 2224
 2225	if ((event->attr.aux_pause || event->attr.aux_resume) &&
 2226	    !(group_leader->pmu->capabilities & PERF_PMU_CAP_AUX_PAUSE))
 2227		return 0;
 2228
 2229	if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux)
 2230		return 0;
 2231
 2232	if (!atomic_long_inc_not_zero(&group_leader->refcount))
 2233		return 0;
 2234
 2235	/*
 2236	 * Link aux_outputs to their aux event; this is undone in
 2237	 * perf_group_detach() by perf_put_aux_event(). When the
 2238	 * group in torn down, the aux_output events loose their
 2239	 * link to the aux_event and can't schedule any more.
 2240	 */
 2241	event->aux_event = group_leader;
 2242
 2243	return 1;
 2244}
 2245
 2246static inline struct list_head *get_event_list(struct perf_event *event)
 2247{
 2248	return event->attr.pinned ? &event->pmu_ctx->pinned_active :
 2249				    &event->pmu_ctx->flexible_active;
 2250}
 2251
 2252static void perf_group_detach(struct perf_event *event)
 2253{
 2254	struct perf_event *leader = event->group_leader;
 2255	struct perf_event *sibling, *tmp;
 2256	struct perf_event_context *ctx = event->ctx;
 2257
 2258	lockdep_assert_held(&ctx->lock);
 2259
 2260	/*
 2261	 * We can have double detach due to exit/hot-unplug + close.
 2262	 */
 2263	if (!(event->attach_state & PERF_ATTACH_GROUP))
 2264		return;
 2265
 2266	event->attach_state &= ~PERF_ATTACH_GROUP;
 2267
 2268	perf_put_aux_event(event);
 2269
 2270	/*
 2271	 * If this is a sibling, remove it from its group.
 2272	 */
 2273	if (leader != event) {
 2274		list_del_init(&event->sibling_list);
 2275		event->group_leader->nr_siblings--;
 2276		event->group_leader->group_generation++;
 2277		goto out;
 2278	}
 2279
 2280	/*
 2281	 * If this was a group event with sibling events then
 2282	 * upgrade the siblings to singleton events by adding them
 2283	 * to whatever list we are on.
 2284	 */
 2285	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) {
 2286
 2287		/*
 2288		 * Events that have PERF_EV_CAP_SIBLING require being part of
 2289		 * a group and cannot exist on their own, schedule them out
 2290		 * and move them into the ERROR state. Also see
 2291		 * _perf_event_enable(), it will not be able to recover this
 2292		 * ERROR state.
 2293		 */
 2294		if (sibling->event_caps & PERF_EV_CAP_SIBLING)
 2295			__event_disable(sibling, ctx, PERF_EVENT_STATE_ERROR);
 2296
 2297		sibling->group_leader = sibling;
 2298		list_del_init(&sibling->sibling_list);
 2299
 2300		/* Inherit group flags from the previous leader */
 2301		sibling->group_caps = event->group_caps;
 2302
 2303		if (sibling->attach_state & PERF_ATTACH_CONTEXT) {
 2304			add_event_to_groups(sibling, event->ctx);
 2305
 2306			if (sibling->state == PERF_EVENT_STATE_ACTIVE)
 2307				list_add_tail(&sibling->active_list, get_event_list(sibling));
 2308		}
 2309
 2310		WARN_ON_ONCE(sibling->ctx != event->ctx);
 2311	}
 2312
 2313out:
 2314	for_each_sibling_event(tmp, leader)
 2315		perf_event__header_size(tmp);
 2316
 2317	perf_event__header_size(leader);
 2318}
 2319
 2320static void perf_child_detach(struct perf_event *event)
 2321{
 2322	struct perf_event *parent_event = event->parent;
 2323
 2324	if (!(event->attach_state & PERF_ATTACH_CHILD))
 2325		return;
 2326
 2327	event->attach_state &= ~PERF_ATTACH_CHILD;
 2328
 2329	if (WARN_ON_ONCE(!parent_event))
 2330		return;
 2331
 2332	/*
 2333	 * Can't check this from an IPI, the holder is likey another CPU.
 2334	 *
 2335	lockdep_assert_held(&parent_event->child_mutex);
 2336	 */
 2337
 2338	list_del_init(&event->child_list);
 2339}
 2340
 2341static bool is_orphaned_event(struct perf_event *event)
 2342{
 2343	return event->state == PERF_EVENT_STATE_DEAD;
 2344}
 2345
 2346static inline int
 2347event_filter_match(struct perf_event *event)
 2348{
 2349	return (event->cpu == -1 || event->cpu == smp_processor_id()) &&
 2350	       perf_cgroup_match(event);
 2351}
 2352
 2353static inline bool is_event_in_freq_mode(struct perf_event *event)
 2354{
 2355	return event->attr.freq && event->attr.sample_freq;
 2356}
 2357
 2358static void
 2359event_sched_out(struct perf_event *event, struct perf_event_context *ctx)
 2360{
 2361	struct perf_event_pmu_context *epc = event->pmu_ctx;
 2362	struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu);
 2363	enum perf_event_state state = PERF_EVENT_STATE_INACTIVE;
 2364
 2365	// XXX cpc serialization, probably per-cpu IRQ disabled
 2366
 2367	WARN_ON_ONCE(event->ctx != ctx);
 2368	lockdep_assert_held(&ctx->lock);
 2369
 2370	if (event->state != PERF_EVENT_STATE_ACTIVE)
 2371		return;
 2372
 2373	/*
 2374	 * Asymmetry; we only schedule events _IN_ through ctx_sched_in(), but
 2375	 * we can schedule events _OUT_ individually through things like
 2376	 * __perf_remove_from_context().
 2377	 */
 2378	list_del_init(&event->active_list);
 2379
 2380	perf_pmu_disable(event->pmu);
 2381
 2382	event->pmu->del(event, 0);
 2383	event->oncpu = -1;
 2384
 2385	if (event->pending_disable) {
 2386		event->pending_disable = 0;
 2387		perf_cgroup_event_disable(event, ctx);
 2388		state = PERF_EVENT_STATE_OFF;
 2389	}
 2390
 2391	perf_event_set_state(event, state);
 2392
 2393	if (!is_software_event(event))
 2394		cpc->active_oncpu--;
 2395	if (is_event_in_freq_mode(event)) {
 2396		ctx->nr_freq--;
 2397		epc->nr_freq--;
 2398	}
 2399	if (event->attr.exclusive || !cpc->active_oncpu)
 2400		cpc->exclusive = 0;
 2401
 2402	perf_pmu_enable(event->pmu);
 2403}
 2404
 2405static void
 2406group_sched_out(struct perf_event *group_event, struct perf_event_context *ctx)
 2407{
 2408	struct perf_event *event;
 2409
 2410	if (group_event->state != PERF_EVENT_STATE_ACTIVE)
 2411		return;
 2412
 2413	perf_assert_pmu_disabled(group_event->pmu_ctx->pmu);
 2414
 2415	event_sched_out(group_event, ctx);
 2416
 2417	/*
 2418	 * Schedule out siblings (if any):
 2419	 */
 2420	for_each_sibling_event(event, group_event)
 2421		event_sched_out(event, ctx);
 2422}
 2423
 2424static inline void
 2425__ctx_time_update(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx, bool final)
 2426{
 2427	if (ctx->is_active & EVENT_TIME) {
 2428		if (ctx->is_active & EVENT_FROZEN)
 2429			return;
 2430		update_context_time(ctx);
 2431		update_cgrp_time_from_cpuctx(cpuctx, final);
 2432	}
 2433}
 2434
 2435static inline void
 2436ctx_time_update(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx)
 2437{
 2438	__ctx_time_update(cpuctx, ctx, false);
 2439}
 2440
 2441/*
 2442 * To be used inside perf_ctx_lock() / perf_ctx_unlock(). Lasts until perf_ctx_unlock().
 2443 */
 2444static inline void
 2445ctx_time_freeze(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx)
 2446{
 2447	ctx_time_update(cpuctx, ctx);
 2448	if (ctx->is_active & EVENT_TIME)
 2449		ctx->is_active |= EVENT_FROZEN;
 2450}
 2451
 2452static inline void
 2453ctx_time_update_event(struct perf_event_context *ctx, struct perf_event *event)
 2454{
 2455	if (ctx->is_active & EVENT_TIME) {
 2456		if (ctx->is_active & EVENT_FROZEN)
 2457			return;
 2458		update_context_time(ctx);
 2459		update_cgrp_time_from_event(event);
 2460	}
 2461}
 2462
 2463#define DETACH_GROUP	0x01UL
 2464#define DETACH_CHILD	0x02UL
 2465#define DETACH_EXIT	0x04UL
 2466#define DETACH_REVOKE	0x08UL
 2467#define DETACH_DEAD	0x10UL
 2468
 2469/*
 2470 * Cross CPU call to remove a performance event
 2471 *
 2472 * We disable the event on the hardware level first. After that we
 2473 * remove it from the context list.
 2474 */
 2475static void
 2476__perf_remove_from_context(struct perf_event *event,
 2477			   struct perf_cpu_context *cpuctx,
 2478			   struct perf_event_context *ctx,
 2479			   void *info)
 2480{
 2481	struct perf_event_pmu_context *pmu_ctx = event->pmu_ctx;
 2482	enum perf_event_state state = PERF_EVENT_STATE_OFF;
 2483	unsigned long flags = (unsigned long)info;
 2484
 2485	ctx_time_update(cpuctx, ctx);
 2486
 2487	/*
 2488	 * Ensure event_sched_out() switches to OFF, at the very least
 2489	 * this avoids raising perf_pending_task() at this time.
 2490	 */
 2491	if (flags & DETACH_EXIT)
 2492		state = PERF_EVENT_STATE_EXIT;
 2493	if (flags & DETACH_REVOKE)
 2494		state = PERF_EVENT_STATE_REVOKED;
 2495	if (flags & DETACH_DEAD)
 2496		state = PERF_EVENT_STATE_DEAD;
 2497
 2498	event_sched_out(event, ctx);
 2499
 2500	if (event->state > PERF_EVENT_STATE_OFF)
 2501		perf_cgroup_event_disable(event, ctx);
 2502
 2503	perf_event_set_state(event, min(event->state, state));
 2504
 2505	if (flags & DETACH_GROUP)
 2506		perf_group_detach(event);
 2507	if (flags & DETACH_CHILD)
 2508		perf_child_detach(event);
 2509	list_del_event(event, ctx);
 2510
 2511	if (!pmu_ctx->nr_events) {
 2512		pmu_ctx->rotate_necessary = 0;
 2513
 2514		if (ctx->task && ctx->is_active) {
 2515			struct perf_cpu_pmu_context *cpc = this_cpc(pmu_ctx->pmu);
 2516
 2517			WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);
 2518			cpc->task_epc = NULL;
 2519		}
 2520	}
 2521
 2522	if (!ctx->nr_events && ctx->is_active) {
 2523		if (ctx == &cpuctx->ctx)
 2524			update_cgrp_time_from_cpuctx(cpuctx, true);
 2525
 2526		ctx->is_active = 0;
 2527		if (ctx->task) {
 2528			WARN_ON_ONCE(cpuctx->task_ctx != ctx);
 2529			cpuctx->task_ctx = NULL;
 2530		}
 2531	}
 2532}
 2533
 2534/*
 2535 * Remove the event from a task's (or a CPU's) list of events.
 2536 *
 2537 * If event->ctx is a cloned context, callers must make sure that
 2538 * every task struct that event->ctx->task could possibly point to
 2539 * remains valid.  This is OK when called from perf_release since
 2540 * that only calls us on the top-level context, which can't be a clone.
 2541 * When called from perf_event_exit_task, it's OK because the
 2542 * context has been detached from its task.
 2543 */
 2544static void perf_remove_from_context(struct perf_event *event, unsigned long flags)
 2545{
 2546	struct perf_event_context *ctx = event->ctx;
 2547
 2548	lockdep_assert_held(&ctx->mutex);
 2549
 2550	/*
 2551	 * Because of perf_event_exit_task(), perf_remove_from_context() ought
 2552	 * to work in the face of TASK_TOMBSTONE, unlike every other
 2553	 * event_function_call() user.
 2554	 */
 2555	raw_spin_lock_irq(&ctx->lock);
 2556	if (!ctx->is_active) {
 2557		__perf_remove_from_context(event, this_cpu_ptr(&perf_cpu_context),
 2558					   ctx, (void *)flags);
 2559		raw_spin_unlock_irq(&ctx->lock);
 2560		return;
 2561	}
 2562	raw_spin_unlock_irq(&ctx->lock);
 2563
 2564	event_function_call(event, __perf_remove_from_context, (void *)flags);
 2565}
 2566
 2567static void __event_disable(struct perf_event *event,
 2568			    struct perf_event_context *ctx,
 2569			    enum perf_event_state state)
 2570{
 2571	event_sched_out(event, ctx);
 2572	perf_cgroup_event_disable(event, ctx);
 2573	perf_event_set_state(event, state);
 2574}
 2575
 2576/*
 2577 * Cross CPU call to disable a performance event
 2578 */
 2579static void __perf_event_disable(struct perf_event *event,
 2580				 struct perf_cpu_context *cpuctx,
 2581				 struct perf_event_context *ctx,
 2582				 void *info)
 2583{
 2584	if (event->state < PERF_EVENT_STATE_INACTIVE)
 2585		return;
 2586
 2587	perf_pmu_disable(event->pmu_ctx->pmu);
 2588	ctx_time_update_event(ctx, event);
 2589
 2590	/*
 2591	 * When disabling a group leader, the whole group becomes ineligible
 2592	 * to run, so schedule out the full group.
 2593	 */
 2594	if (event == event->group_leader)
 2595		group_sched_out(event, ctx);
 2596
 2597	/*
 2598	 * But only mark the leader OFF; the siblings will remain
 2599	 * INACTIVE.
 2600	 */
 2601	__event_disable(event, ctx, PERF_EVENT_STATE_OFF);
 2602
 2603	perf_pmu_enable(event->pmu_ctx->pmu);
 2604}
 2605
 2606/*
 2607 * Disable an event.
 2608 *
 2609 * If event->ctx is a cloned context, callers must make sure that
 2610 * every task struct that event->ctx->task could possibly point to
 2611 * remains valid.  This condition is satisfied when called through
 2612 * perf_event_for_each_child or perf_event_for_each because they
 2613 * hold the top-level event's child_mutex, so any descendant that
 2614 * goes to exit will block in perf_event_exit_event().
 2615 *
 2616 * When called from perf_pending_disable it's OK because event->ctx
 2617 * is the current context on this CPU and preemption is disabled,
 2618 * hence we can't get into perf_event_task_sched_out for this context.
 2619 */
 2620static void _perf_event_disable(struct perf_event *event)
 2621{
 2622	struct perf_event_context *ctx = event->ctx;
 2623
 2624	raw_spin_lock_irq(&ctx->lock);
 2625	if (event->state <= PERF_EVENT_STATE_OFF) {
 2626		raw_spin_unlock_irq(&ctx->lock);
 2627		return;
 2628	}
 2629	raw_spin_unlock_irq(&ctx->lock);
 2630
 2631	event_function_call(event, __perf_event_disable, NULL);
 2632}
 2633
 2634void perf_event_disable_local(struct perf_event *event)
 2635{
 2636	event_function_local(event, __perf_event_disable, NULL);
 2637}
 2638
 2639/*
 2640 * Strictly speaking kernel users cannot create groups and therefore this
 2641 * interface does not need the perf_event_ctx_lock() magic.
 2642 */
 2643void perf_event_disable(struct perf_event *event)
 2644{
 2645	struct perf_event_context *ctx;
 2646
 2647	ctx = perf_event_ctx_lock(event);
 2648	_perf_event_disable(event);
 2649	perf_event_ctx_unlock(event, ctx);
 2650}
 2651EXPORT_SYMBOL_GPL(perf_event_disable);
 2652
 2653void perf_event_disable_inatomic(struct perf_event *event)
 2654{
 2655	event->pending_disable = 1;
 2656	irq_work_queue(&event->pending_disable_irq);
 2657}
 2658
 2659#define MAX_INTERRUPTS (~0ULL)
 2660
 2661static void perf_log_throttle(struct perf_event *event, int enable);
 2662static void perf_log_itrace_start(struct perf_event *event);
 2663
 2664static void perf_event_unthrottle(struct perf_event *event, bool start)
 2665{
 2666	if (event->state != PERF_EVENT_STATE_ACTIVE)
 2667		return;
 2668
 2669	event->hw.interrupts = 0;
 2670	if (start)
 2671		event->pmu->start(event, 0);
 2672	if (event == event->group_leader)
 2673		perf_log_throttle(event, 1);
 2674}
 2675
 2676static void perf_event_throttle(struct perf_event *event)
 2677{
 2678	if (event->state != PERF_EVENT_STATE_ACTIVE)
 2679		return;
 2680
 2681	event->hw.interrupts = MAX_INTERRUPTS;
 2682	event->pmu->stop(event, 0);
 2683	if (event == event->group_leader)
 2684		perf_log_throttle(event, 0);
 2685}
 2686
 2687static void perf_event_unthrottle_group(struct perf_event *event, bool skip_start_event)
 2688{
 2689	struct perf_event *sibling, *leader = event->group_leader;
 2690
 2691	perf_event_unthrottle(leader, skip_start_event ? leader != event : true);
 2692	for_each_sibling_event(sibling, leader)
 2693		perf_event_unthrottle(sibling, skip_start_event ? sibling != event : true);
 2694}
 2695
 2696static void perf_event_throttle_group(struct perf_event *event)
 2697{
 2698	struct perf_event *sibling, *leader = event->group_leader;
 2699
 2700	perf_event_throttle(leader);
 2701	for_each_sibling_event(sibling, leader)
 2702		perf_event_throttle(sibling);
 2703}
 2704
 2705static int
 2706event_sched_in(struct perf_event *event, struct perf_event_context *ctx)
 2707{
 2708	struct perf_event_pmu_context *epc = event->pmu_ctx;
 2709	struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu);
 2710	int ret = 0;
 2711
 2712	WARN_ON_ONCE(event->ctx != ctx);
 2713
 2714	lockdep_assert_held(&ctx->lock);
 2715
 2716	if (event->state <= PERF_EVENT_STATE_OFF)
 2717		return 0;
 2718
 2719	WRITE_ONCE(event->oncpu, smp_processor_id());
 2720	/*
 2721	 * Order event::oncpu write to happen before the ACTIVE state is
 2722	 * visible. This allows perf_event_{stop,read}() to observe the correct
 2723	 * ->oncpu if it sees ACTIVE.
 2724	 */
 2725	smp_wmb();
 2726	perf_event_set_state(event, PERF_EVENT_STATE_ACTIVE);
 2727
 2728	/*
 2729	 * Unthrottle events, since we scheduled we might have missed several
 2730	 * ticks already, also for a heavily scheduling task there is little
 2731	 * guarantee it'll get a tick in a timely manner.
 2732	 */
 2733	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS))
 2734		perf_event_unthrottle(event, false);
 2735
 2736	perf_pmu_disable(event->pmu);
 2737
 2738	perf_log_itrace_start(event);
 2739
 2740	if (event->pmu->add(event, PERF_EF_START)) {
 2741		perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
 2742		event->oncpu = -1;
 2743		ret = -EAGAIN;
 2744		goto out;
 2745	}
 2746
 2747	if (!is_software_event(event))
 2748		cpc->active_oncpu++;
 2749	if (is_event_in_freq_mode(event)) {
 2750		ctx->nr_freq++;
 2751		epc->nr_freq++;
 2752	}
 2753	if (event->attr.exclusive)
 2754		cpc->exclusive = 1;
 2755
 2756out:
 2757	perf_pmu_enable(event->pmu);
 2758
 2759	return ret;
 2760}
 2761
 2762static int
 2763group_sched_in(struct perf_event *group_event, struct perf_event_context *ctx)
 2764{
 2765	struct perf_event *event, *partial_group = NULL;
 2766	struct pmu *pmu = group_event->pmu_ctx->pmu;
 2767
 2768	if (group_event->state == PERF_EVENT_STATE_OFF)
 2769		return 0;
 2770
 2771	pmu->start_txn(pmu, PERF_PMU_TXN_ADD);
 2772
 2773	if (event_sched_in(group_event, ctx))
 2774		goto error;
 2775
 2776	/*
 2777	 * Schedule in siblings as one group (if any):
 2778	 */
 2779	for_each_sibling_event(event, group_event) {
 2780		if (event_sched_in(event, ctx)) {
 2781			partial_group = event;
 2782			goto group_error;
 2783		}
 2784	}
 2785
 2786	if (!pmu->commit_txn(pmu))
 2787		return 0;
 2788
 2789group_error:
 2790	/*
 2791	 * Groups can be scheduled in as one unit only, so undo any
 2792	 * partial group before returning:
 2793	 * The events up to the failed event are scheduled out normally.
 2794	 */
 2795	for_each_sibling_event(event, group_event) {
 2796		if (event == partial_group)
 2797			break;
 2798
 2799		event_sched_out(event, ctx);
 2800	}
 2801	event_sched_out(group_event, ctx);
 2802
 2803error:
 2804	pmu->cancel_txn(pmu);
 2805	return -EAGAIN;
 2806}
 2807
 2808/*
 2809 * Work out whether we can put this event group on the CPU now.
 2810 */
 2811static int group_can_go_on(struct perf_event *event, int can_add_hw)
 2812{
 2813	struct perf_event_pmu_context *epc = event->pmu_ctx;
 2814	struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu);
 2815
 2816	/*
 2817	 * Groups consisting entirely of software events can always go on.
 2818	 */
 2819	if (event->group_caps & PERF_EV_CAP_SOFTWARE)
 2820		return 1;
 2821	/*
 2822	 * If an exclusive group is already on, no other hardware
 2823	 * events can go on.
 2824	 */
 2825	if (cpc->exclusive)
 2826		return 0;
 2827	/*
 2828	 * If this group is exclusive and there are already
 2829	 * events on the CPU, it can't go on.
 2830	 */
 2831	if (event->attr.exclusive && !list_empty(get_event_list(event)))
 2832		return 0;
 2833	/*
 2834	 * Otherwise, try to add it if all previous groups were able
 2835	 * to go on.
 2836	 */
 2837	return can_add_hw;
 2838}
 2839
 2840static void add_event_to_ctx(struct perf_event *event,
 2841			       struct perf_event_context *ctx)
 2842{
 2843	list_add_event(event, ctx);
 2844	perf_group_attach(event);
 2845}
 2846
 2847static void task_ctx_sched_out(struct perf_event_context *ctx,
 2848			       struct pmu *pmu,
 2849			       enum event_type_t event_type)
 2850{
 2851	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
 2852
 2853	if (!cpuctx->task_ctx)
 2854		return;
 2855
 2856	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
 2857		return;
 2858
 2859	ctx_sched_out(ctx, pmu, event_type);
 2860}
 2861
 2862static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
 2863				struct perf_event_context *ctx,
 2864				struct pmu *pmu)
 2865{
 2866	ctx_sched_in(&cpuctx->ctx, pmu, EVENT_PINNED);
 2867	if (ctx)
 2868		 ctx_sched_in(ctx, pmu, EVENT_PINNED);
 2869	ctx_sched_in(&cpuctx->ctx, pmu, EVENT_FLEXIBLE);
 2870	if (ctx)
 2871		 ctx_sched_in(ctx, pmu, EVENT_FLEXIBLE);
 2872}
 2873
 2874/*
 2875 * We want to maintain the following priority of scheduling:
 2876 *  - CPU pinned (EVENT_CPU | EVENT_PINNED)
 2877 *  - task pinned (EVENT_PINNED)
 2878 *  - CPU flexible (EVENT_CPU | EVENT_FLEXIBLE)
 2879 *  - task flexible (EVENT_FLEXIBLE).
 2880 *
 2881 * In order to avoid unscheduling and scheduling back in everything every
 2882 * time an event is added, only do it for the groups of equal priority and
 2883 * below.
 2884 *
 2885 * This can be called after a batch operation on task events, in which case
 2886 * event_type is a bit mask of the types of events involved. For CPU events,
 2887 * event_type is only either EVENT_PINNED or EVENT_FLEXIBLE.
 2888 */
 2889static void ctx_resched(struct perf_cpu_context *cpuctx,
 2890			struct perf_event_context *task_ctx,
 2891			struct pmu *pmu, enum event_type_t event_type)
 2892{
 2893	bool cpu_event = !!(event_type & EVENT_CPU);
 2894	struct perf_event_pmu_context *epc;
 2895
 2896	/*
 2897	 * If pinned groups are involved, flexible groups also need to be
 2898	 * scheduled out.
 2899	 */
 2900	if (event_type & EVENT_PINNED)
 2901		event_type |= EVENT_FLEXIBLE;
 2902
 2903	event_type &= EVENT_ALL;
 2904
 2905	for_each_epc(epc, &cpuctx->ctx, pmu, false)
 2906		perf_pmu_disable(epc->pmu);
 2907
 2908	if (task_ctx) {
 2909		for_each_epc(epc, task_ctx, pmu, false)
 2910			perf_pmu_disable(epc->pmu);
 2911
 2912		task_ctx_sched_out(task_ctx, pmu, event_type);
 2913	}
 2914
 2915	/*
 2916	 * Decide which cpu ctx groups to schedule out based on the types
 2917	 * of events that caused rescheduling:
 2918	 *  - EVENT_CPU: schedule out corresponding groups;
 2919	 *  - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups;
 2920	 *  - otherwise, do nothing more.
 2921	 */
 2922	if (cpu_event)
 2923		ctx_sched_out(&cpuctx->ctx, pmu, event_type);
 2924	else if (event_type & EVENT_PINNED)
 2925		ctx_sched_out(&cpuctx->ctx, pmu, EVENT_FLEXIBLE);
 2926
 2927	perf_event_sched_in(cpuctx, task_ctx, pmu);
 2928
 2929	for_each_epc(epc, &cpuctx->ctx, pmu, false)
 2930		perf_pmu_enable(epc->pmu);
 2931
 2932	if (task_ctx) {
 2933		for_each_epc(epc, task_ctx, pmu, false)
 2934			perf_pmu_enable(epc->pmu);
 2935	}
 2936}
 2937
 2938void perf_pmu_resched(struct pmu *pmu)
 2939{
 2940	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
 2941	struct perf_event_context *task_ctx = cpuctx->task_ctx;
 2942
 2943	perf_ctx_lock(cpuctx, task_ctx);
 2944	ctx_resched(cpuctx, task_ctx, pmu, EVENT_ALL|EVENT_CPU);
 2945	perf_ctx_unlock(cpuctx, task_ctx);
 2946}
 2947
 2948/*
 2949 * Cross CPU call to install and enable a performance event
 2950 *
 2951 * Very similar to remote_function() + event_function() but cannot assume that
 2952 * things like ctx->is_active and cpuctx->task_ctx are set.
 2953 */
 2954static int  __perf_install_in_context(void *info)
 2955{
 2956	struct perf_event *event = info;
 2957	struct perf_event_context *ctx = event->ctx;
 2958	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
 2959	struct perf_event_context *task_ctx = cpuctx->task_ctx;
 2960	bool reprogram = true;
 2961	int ret = 0;
 2962
 2963	raw_spin_lock(&cpuctx->ctx.lock);
 2964	if (ctx->task) {
 2965		raw_spin_lock(&ctx->lock);
 2966		task_ctx = ctx;
 2967
 2968		reprogram = (ctx->task == current);
 2969
 2970		/*
 2971		 * If the task is running, it must be running on this CPU,
 2972		 * otherwise we cannot reprogram things.
 2973		 *
 2974		 * If its not running, we don't care, ctx->lock will
 2975		 * serialize against it becoming runnable.
 2976		 */
 2977		if (task_curr(ctx->task) && !reprogram) {
 2978			ret = -ESRCH;
 2979			goto unlock;
 2980		}
 2981
 2982		WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx);
 2983	} else if (task_ctx) {
 2984		raw_spin_lock(&task_ctx->lock);
 2985	}
 2986
 2987#ifdef CONFIG_CGROUP_PERF
 2988	if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) {
 2989		/*
 2990		 * If the current cgroup doesn't match the event's
 2991		 * cgroup, we should not try to schedule it.
 2992		 */
 2993		struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx);
 2994		reprogram = cgroup_is_descendant(cgrp->css.cgroup,
 2995					event->cgrp->css.cgroup);
 2996	}
 2997#endif
 2998
 2999	if (reprogram) {
 3000		ctx_time_freeze(cpuctx, ctx);
 3001		add_event_to_ctx(event, ctx);
 3002		ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu,
 3003			    get_event_type(event));
 3004	} else {
 3005		add_event_to_ctx(event, ctx);
 3006	}
 3007
 3008unlock:
 3009	perf_ctx_unlock(cpuctx, task_ctx);
 3010
 3011	return ret;
 3012}
 3013
 3014static bool exclusive_event_installable(struct perf_event *event,
 3015					struct perf_event_context *ctx);
 3016
 3017/*
 3018 * Attach a performance event to a context.
 3019 *
 3020 * Very similar to event_function_call, see comment there.
 3021 */
 3022static void
 3023perf_install_in_context(struct perf_event_context *ctx,
 3024			struct perf_event *event,
 3025			int cpu)
 3026{
 3027	struct task_struct *task = READ_ONCE(ctx->task);
 3028
 3029	lockdep_assert_held(&ctx->mutex);
 3030
 3031	WARN_ON_ONCE(!exclusive_event_installable(event, ctx));
 3032
 3033	if (event->cpu != -1)
 3034		WARN_ON_ONCE(event->cpu != cpu);
 3035
 3036	/*
 3037	 * Ensures that if we can observe event->ctx, both the event and ctx
 3038	 * will be 'complete'. See perf_iterate_sb_cpu().
 3039	 */
 3040	smp_store_release(&event->ctx, ctx);
 3041
 3042	/*
 3043	 * perf_event_attr::disabled events will not run and can be initialized
 3044	 * without IPI. Except when this is the first event for the context, in
 3045	 * that case we need the magic of the IPI to set ctx->is_active.
 3046	 *
 3047	 * The IOC_ENABLE that is sure to follow the creation of a disabled
 3048	 * event will issue the IPI and reprogram the hardware.
 3049	 */
 3050	if (__perf_effective_state(event) == PERF_EVENT_STATE_OFF &&
 3051	    ctx->nr_events && !is_cgroup_event(event)) {
 3052		raw_spin_lock_irq(&ctx->lock);
 3053		if (ctx->task == TASK_TOMBSTONE) {
 3054			raw_spin_unlock_irq(&ctx->lock);
 3055			return;
 3056		}
 3057		add_event_to_ctx(event, ctx);
 3058		raw_spin_unlock_irq(&ctx->lock);
 3059		return;
 3060	}
 3061
 3062	if (!task) {
 3063		cpu_function_call(cpu, __perf_install_in_context, event);
 3064		return;
 3065	}
 3066
 3067	/*
 3068	 * Should not happen, we validate the ctx is still alive before calling.
 3069	 */
 3070	if (WARN_ON_ONCE(task == TASK_TOMBSTONE))
 3071		return;
 3072
 3073	/*
 3074	 * Installing events is tricky because we cannot rely on ctx->is_active
 3075	 * to be set in case this is the nr_events 0 -> 1 transition.
 3076	 *
 3077	 * Instead we use task_curr(), which tells us if the task is running.
 3078	 * However, since we use task_curr() outside of rq::lock, we can race
 3079	 * against the actual state. This means the result can be wrong.
 3080	 *
 3081	 * If we get a false positive, we retry, this is harmless.
 3082	 *
 3083	 * If we get a false negative, things are complicated. If we are after
 3084	 * perf_event_context_sched_in() ctx::lock will serialize us, and the
 3085	 * value must be correct. If we're before, it doesn't matter since
 3086	 * perf_event_context_sched_in() will program the counter.
 3087	 *
 3088	 * However, this hinges on the remote context switch having observed
 3089	 * our task->perf_event_ctxp[] store, such that it will in fact take
 3090	 * ctx::lock in perf_event_context_sched_in().
 3091	 *
 3092	 * We do this by task_function_call(), if the IPI fails to hit the task
 3093	 * we know any future context switch of task must see the
 3094	 * perf_event_ctpx[] store.
 3095	 */
 3096
 3097	/*
 3098	 * This smp_mb() orders the task->perf_event_ctxp[] store with the
 3099	 * task_cpu() load, such that if the IPI then does not find the task
 3100	 * running, a future context switch of that task must observe the
 3101	 * store.
 3102	 */
 3103	smp_mb();
 3104again:
 3105	if (!task_function_call(task, __perf_install_in_context, event))
 3106		return;
 3107
 3108	raw_spin_lock_irq(&ctx->lock);
 3109	task = ctx->task;
 3110	if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) {
 3111		/*
 3112		 * Cannot happen because we already checked above (which also
 3113		 * cannot happen), and we hold ctx->mutex, which serializes us
 3114		 * against perf_event_exit_task_context().
 3115		 */
 3116		raw_spin_unlock_irq(&ctx->lock);
 3117		return;
 3118	}
 3119	/*
 3120	 * If the task is not running, ctx->lock will avoid it becoming so,
 3121	 * thus we can safely install the event.
 3122	 */
 3123	if (task_curr(task)) {
 3124		raw_spin_unlock_irq(&ctx->lock);
 3125		goto again;
 3126	}
 3127	add_event_to_ctx(event, ctx);
 3128	raw_spin_unlock_irq(&ctx->lock);
 3129}
 3130
 3131/*
 3132 * Cross CPU call to enable a performance event
 3133 */
 3134static void __perf_event_enable(struct perf_event *event,
 3135				struct perf_cpu_context *cpuctx,
 3136				struct perf_event_context *ctx,
 3137				void *info)
 3138{
 3139	struct perf_event *leader = event->group_leader;
 3140	struct perf_event_context *task_ctx;
 3141
 3142	if (event->state >= PERF_EVENT_STATE_INACTIVE ||
 3143	    event->state <= PERF_EVENT_STATE_ERROR)
 3144		return;
 3145
 3146	ctx_time_freeze(cpuctx, ctx);
 3147
 3148	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
 3149	perf_cgroup_event_enable(event, ctx);
 3150
 3151	if (!ctx->is_active)
 3152		return;
 3153
 3154	if (!event_filter_match(event))
 3155		return;
 3156
 3157	/*
 3158	 * If the event is in a group and isn't the group leader,
 3159	 * then don't put it on unless the group is on.
 3160	 */
 3161	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
 3162		return;
 3163
 3164	task_ctx = cpuctx->task_ctx;
 3165	if (ctx->task)
 3166		WARN_ON_ONCE(task_ctx != ctx);
 3167
 3168	ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu, get_event_type(event));
 3169}
 3170
 3171/*
 3172 * Enable an event.
 3173 *
 3174 * If event->ctx is a cloned context, callers must make sure that
 3175 * every task struct that event->ctx->task could possibly point to
 3176 * remains valid.  This condition is satisfied when called through
 3177 * perf_event_for_each_child or perf_event_for_each as described
 3178 * for perf_event_disable.
 3179 */
 3180static void _perf_event_enable(struct perf_event *event)
 3181{
 3182	struct perf_event_context *ctx = event->ctx;
 3183
 3184	raw_spin_lock_irq(&ctx->lock);
 3185	if (event->state >= PERF_EVENT_STATE_INACTIVE ||
 3186	    event->state <  PERF_EVENT_STATE_ERROR) {
 3187out:
 3188		raw_spin_unlock_irq(&ctx->lock);
 3189		return;
 3190	}
 3191
 3192	/*
 3193	 * If the event is in error state, clear that first.
 3194	 *
 3195	 * That way, if we see the event in error state below, we know that it
 3196	 * has gone back into error state, as distinct from the task having
 3197	 * been scheduled away before the cross-call arrived.
 3198	 */
 3199	if (event->state == PERF_EVENT_STATE_ERROR) {
 3200		/*
 3201		 * Detached SIBLING events cannot leave ERROR state.
 3202		 */
 3203		if (event->event_caps & PERF_EV_CAP_SIBLING &&
 3204		    event->group_leader == event)
 3205			goto out;
 3206
 3207		event->state = PERF_EVENT_STATE_OFF;
 3208	}
 3209	raw_spin_unlock_irq(&ctx->lock);
 3210
 3211	event_function_call(event, __perf_event_enable, NULL);
 3212}
 3213
 3214/*
 3215 * See perf_event_disable();
 3216 */
 3217void perf_event_enable(struct perf_event *event)
 3218{
 3219	struct perf_event_context *ctx;
 3220
 3221	ctx = perf_event_ctx_lock(event);
 3222	_perf_event_enable(event);
 3223	perf_event_ctx_unlock(event, ctx);
 3224}
 3225EXPORT_SYMBOL_GPL(perf_event_enable);
 3226
 3227struct stop_event_data {
 3228	struct perf_event	*event;
 3229	unsigned int		restart;
 3230};
 3231
 3232static int __perf_event_stop(void *info)
 3233{
 3234	struct stop_event_data *sd = info;
 3235	struct perf_event *event = sd->event;
 3236
 3237	/* if it's already INACTIVE, do nothing */
 3238	if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
 3239		return 0;
 3240
 3241	/* matches smp_wmb() in event_sched_in() */
 3242	smp_rmb();
 3243
 3244	/*
 3245	 * There is a window with interrupts enabled before we get here,
 3246	 * so we need to check again lest we try to stop another CPU's event.
 3247	 */
 3248	if (READ_ONCE(event->oncpu) != smp_processor_id())
 3249		return -EAGAIN;
 3250
 3251	event->pmu->stop(event, PERF_EF_UPDATE);
 3252
 3253	/*
 3254	 * May race with the actual stop (through perf_pmu_output_stop()),
 3255	 * but it is only used for events with AUX ring buffer, and such
 3256	 * events will refuse to restart because of rb::aux_mmap_count==0,
 3257	 * see comments in perf_aux_output_begin().
 3258	 *
 3259	 * Since this is happening on an event-local CPU, no trace is lost
 3260	 * while restarting.
 3261	 */
 3262	if (sd->restart)
 3263		event->pmu->start(event, 0);
 3264
 3265	return 0;
 3266}
 3267
 3268static int perf_event_stop(struct perf_event *event, int restart)
 3269{
 3270	struct stop_event_data sd = {
 3271		.event		= event,
 3272		.restart	= restart,
 3273	};
 3274	int ret = 0;
 3275
 3276	do {
 3277		if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
 3278			return 0;
 3279
 3280		/* matches smp_wmb() in event_sched_in() */
 3281		smp_rmb();
 3282
 3283		/*
 3284		 * We only want to restart ACTIVE events, so if the event goes
 3285		 * inactive here (event->oncpu==-1), there's nothing more to do;
 3286		 * fall through with ret==-ENXIO.
 3287		 */
 3288		ret = cpu_function_call(READ_ONCE(event->oncpu),
 3289					__perf_event_stop, &sd);
 3290	} while (ret == -EAGAIN);
 3291
 3292	return ret;
 3293}
 3294
 3295/*
 3296 * In order to contain the amount of racy and tricky in the address filter
 3297 * configuration management, it is a two part process:
 3298 *
 3299 * (p1) when userspace mappings change as a result of (1) or (2) or (3) below,
 3300 *      we update the addresses of corresponding vmas in
 3301 *	event::addr_filter_ranges array and bump the event::addr_filters_gen;
 3302 * (p2) when an event is scheduled in (pmu::add), it calls
 3303 *      perf_event_addr_filters_sync() which calls pmu::addr_filters_sync()
 3304 *      if the generation has changed since the previous call.
 3305 *
 3306 * If (p1) happens while the event is active, we restart it to force (p2).
 3307 *
 3308 * (1) perf_addr_filters_apply(): adjusting filters' offsets based on
 3309 *     pre-existing mappings, called once when new filters arrive via SET_FILTER
 3310 *     ioctl;
 3311 * (2) perf_addr_filters_adjust(): adjusting filters' offsets based on newly
 3312 *     registered mapping, called for every new mmap(), with mm::mmap_lock down
 3313 *     for reading;
 3314 * (3) perf_event_addr_filters_exec(): clearing filters' offsets in the process
 3315 *     of exec.
 3316 */
 3317void perf_event_addr_filters_sync(struct perf_event *event)
 3318{
 3319	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
 3320
 3321	if (!has_addr_filter(event))
 3322		return;
 3323
 3324	raw_spin_lock(&ifh->lock);
 3325	if (event->addr_filters_gen != event->hw.addr_filters_gen) {
 3326		event->pmu->addr_filters_sync(event);
 3327		event->hw.addr_filters_gen = event->addr_filters_gen;
 3328	}
 3329	raw_spin_unlock(&ifh->lock);
 3330}
 3331EXPORT_SYMBOL_GPL(perf_event_addr_filters_sync);
 3332
 3333static int _perf_event_refresh(struct perf_event *event, int refresh)
 3334{
 3335	/*
 3336	 * not supported on inherited events
 3337	 */
 3338	if (event->attr.inherit || !is_sampling_event(event))
 3339		return -EINVAL;
 3340
 3341	atomic_add(refresh, &event->event_limit);
 3342	_perf_event_enable(event);
 3343
 3344	return 0;
 3345}
 3346
 3347/*
 3348 * See perf_event_disable()
 3349 */
 3350int perf_event_refresh(struct perf_event *event, int refresh)
 3351{
 3352	struct perf_event_context *ctx;
 3353	int ret;
 3354
 3355	ctx = perf_event_ctx_lock(event);
 3356	ret = _perf_event_refresh(event, refresh);
 3357	perf_event_ctx_unlock(event, ctx);
 3358
 3359	return ret;
 3360}
 3361EXPORT_SYMBOL_GPL(perf_event_refresh);
 3362
 3363static int perf_event_modify_breakpoint(struct perf_event *bp,
 3364					 struct perf_event_attr *attr)
 3365{
 3366	int err;
 3367
 3368	_perf_event_disable(bp);
 3369
 3370	err = modify_user_hw_breakpoint_check(bp, attr, true);
 3371
 3372	if (!bp->attr.disabled)
 3373		_perf_event_enable(bp);
 3374
 3375	return err;
 3376}
 3377
 3378/*
 3379 * Copy event-type-independent attributes that may be modified.
 3380 */
 3381static void perf_event_modify_copy_attr(struct perf_event_attr *to,
 3382					const struct perf_event_attr *from)
 3383{
 3384	to->sig_data = from->sig_data;
 3385}
 3386
 3387static int perf_event_modify_attr(struct perf_event *event,
 3388				  struct perf_event_attr *attr)
 3389{
 3390	int (*func)(struct perf_event *, struct perf_event_attr *);
 3391	struct perf_event *child;
 3392	int err;
 3393
 3394	if (event->attr.type != attr->type)
 3395		return -EINVAL;
 3396
 3397	switch (event->attr.type) {
 3398	case PERF_TYPE_BREAKPOINT:
 3399		func = perf_event_modify_breakpoint;
 3400		break;
 3401	default:
 3402		/* Place holder for future additions. */
 3403		return -EOPNOTSUPP;
 3404	}
 3405
 3406	WARN_ON_ONCE(event->ctx->parent_ctx);
 3407
 3408	mutex_lock(&event->child_mutex);
 3409	/*
 3410	 * Event-type-independent attributes must be copied before event-type
 3411	 * modification, which will validate that final attributes match the
 3412	 * source attributes after all relevant attributes have been copied.
 3413	 */
 3414	perf_event_modify_copy_attr(&event->attr, attr);
 3415	err = func(event, attr);
 3416	if (err)
 3417		goto out;
 3418	list_for_each_entry(child, &event->child_list, child_list) {
 3419		perf_event_modify_copy_attr(&child->attr, attr);
 3420		err = func(child, attr);
 3421		if (err)
 3422			goto out;
 3423	}
 3424out:
 3425	mutex_unlock(&event->child_mutex);
 3426	return err;
 3427}
 3428
 3429static void __pmu_ctx_sched_out(struct perf_event_pmu_context *pmu_ctx,
 3430				enum event_type_t event_type)
 3431{
 3432	struct perf_event_context *ctx = pmu_ctx->ctx;
 3433	struct perf_event *event, *tmp;
 3434	struct pmu *pmu = pmu_ctx->pmu;
 3435
 3436	if (ctx->task && !(ctx->is_active & EVENT_ALL)) {
 3437		struct perf_cpu_pmu_context *cpc = this_cpc(pmu);
 3438
 3439		WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);
 3440		cpc->task_epc = NULL;
 3441	}
 3442
 3443	if (!(event_type & EVENT_ALL))
 3444		return;
 3445
 3446	perf_pmu_disable(pmu);
 3447	if (event_type & EVENT_PINNED) {
 3448		list_for_each_entry_safe(event, tmp,
 3449					 &pmu_ctx->pinned_active,
 3450					 active_list)
 3451			group_sched_out(event, ctx);
 3452	}
 3453
 3454	if (event_type & EVENT_FLEXIBLE) {
 3455		list_for_each_entry_safe(event, tmp,
 3456					 &pmu_ctx->flexible_active,
 3457					 active_list)
 3458			group_sched_out(event, ctx);
 3459		/*
 3460		 * Since we cleared EVENT_FLEXIBLE, also clear
 3461		 * rotate_necessary, is will be reset by
 3462		 * ctx_flexible_sched_in() when needed.
 3463		 */
 3464		pmu_ctx->rotate_necessary = 0;
 3465	}
 3466	perf_pmu_enable(pmu);
 3467}
 3468
 3469/*
 3470 * Be very careful with the @pmu argument since this will change ctx state.
 3471 * The @pmu argument works for ctx_resched(), because that is symmetric in
 3472 * ctx_sched_out() / ctx_sched_in() usage and the ctx state ends up invariant.
 3473 *
 3474 * However, if you were to be asymmetrical, you could end up with messed up
 3475 * state, eg. ctx->is_active cleared even though most EPCs would still actually
 3476 * be active.
 3477 */
 3478static void
 3479ctx_sched_out(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type)
 3480{
 3481	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
 3482	struct perf_event_pmu_context *pmu_ctx;
 3483	int is_active = ctx->is_active;
 3484	bool cgroup = event_type & EVENT_CGROUP;
 3485
 3486	event_type &= ~EVENT_CGROUP;
 3487
 3488	lockdep_assert_held(&ctx->lock);
 3489
 3490	if (likely(!ctx->nr_events)) {
 3491		/*
 3492		 * See __perf_remove_from_context().
 3493		 */
 3494		WARN_ON_ONCE(ctx->is_active);
 3495		if (ctx->task)
 3496			WARN_ON_ONCE(cpuctx->task_ctx);
 3497		return;
 3498	}
 3499
 3500	/*
 3501	 * Always update time if it was set; not only when it changes.
 3502	 * Otherwise we can 'forget' to update time for any but the last
 3503	 * context we sched out. For example:
 3504	 *
 3505	 *   ctx_sched_out(.event_type = EVENT_FLEXIBLE)
 3506	 *   ctx_sched_out(.event_type = EVENT_PINNED)
 3507	 *
 3508	 * would only update time for the pinned events.
 3509	 */
 3510	__ctx_time_update(cpuctx, ctx, ctx == &cpuctx->ctx);
 3511
 3512	/*
 3513	 * CPU-release for the below ->is_active store,
 3514	 * see __load_acquire() in perf_event_time_now()
 3515	 */
 3516	barrier();
 3517	ctx->is_active &= ~event_type;
 3518
 3519	if (!(ctx->is_active & EVENT_ALL)) {
 3520		/*
 3521		 * For FROZEN, preserve TIME|FROZEN such that perf_event_time_now()
 3522		 * does not observe a hole. perf_ctx_unlock() will clean up.
 3523		 */
 3524		if (ctx->is_active & EVENT_FROZEN)
 3525			ctx->is_active &= EVENT_TIME_FROZEN;
 3526		else
 3527			ctx->is_active = 0;
 3528	}
 3529
 3530	if (ctx->task) {
 3531		WARN_ON_ONCE(cpuctx->task_ctx != ctx);
 3532		if (!(ctx->is_active & EVENT_ALL))
 3533			cpuctx->task_ctx = NULL;
 3534	}
 3535
 3536	is_active ^= ctx->is_active; /* changed bits */
 3537
 3538	for_each_epc(pmu_ctx, ctx, pmu, cgroup)
 3539		__pmu_ctx_sched_out(pmu_ctx, is_active);
 3540}
 3541
 3542/*
 3543 * Test whether two contexts are equivalent, i.e. whether they have both been
 3544 * cloned from the same version of the same context.
 3545 *
 3546 * Equivalence is measured using a generation number in the context that is
 3547 * incremented on each modification to it; see unclone_ctx(), list_add_event()
 3548 * and list_del_event().
 3549 */
 3550static int context_equiv(struct perf_event_context *ctx1,
 3551			 struct perf_event_context *ctx2)
 3552{
 3553	lockdep_assert_held(&ctx1->lock);
 3554	lockdep_assert_held(&ctx2->lock);
 3555
 3556	/* Pinning disables the swap optimization */
 3557	if (ctx1->pin_count || ctx2->pin_count)
 3558		return 0;
 3559
 3560	/* If ctx1 is the parent of ctx2 */
 3561	if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)
 3562		return 1;
 3563
 3564	/* If ctx2 is the parent of ctx1 */
 3565	if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)
 3566		return 1;
 3567
 3568	/*
 3569	 * If ctx1 and ctx2 have the same parent; we flatten the parent
 3570	 * hierarchy, see perf_event_init_context().
 3571	 */
 3572	if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&
 3573			ctx1->parent_gen == ctx2->parent_gen)
 3574		return 1;
 3575
 3576	/* Unmatched */
 3577	return 0;
 3578}
 3579
 3580static void __perf_event_sync_stat(struct perf_event *event,
 3581				     struct perf_event *next_event)
 3582{
 3583	u64 value;
 3584
 3585	if (!event->attr.inherit_stat)
 3586		return;
 3587
 3588	/*
 3589	 * Update the event value, we cannot use perf_event_read()
 3590	 * because we're in the middle of a context switch and have IRQs
 3591	 * disabled, which upsets smp_call_function_single(), however
 3592	 * we know the event must be on the current CPU, therefore we
 3593	 * don't need to use it.
 3594	 */
 3595	perf_pmu_read(event);
 3596
 3597	perf_event_update_time(event);
 3598
 3599	/*
 3600	 * In order to keep per-task stats reliable we need to flip the event
 3601	 * values when we flip the contexts.
 3602	 */
 3603	value = local64_read(&next_event->count);
 3604	value = local64_xchg(&event->count, value);
 3605	local64_set(&next_event->count, value);
 3606
 3607	swap(event->total_time_enabled, next_event->total_time_enabled);
 3608	swap(event->total_time_running, next_event->total_time_running);
 3609
 3610	/*
 3611	 * Since we swizzled the values, update the user visible data too.
 3612	 */
 3613	perf_event_update_userpage(event);
 3614	perf_event_update_userpage(next_event);
 3615}
 3616
 3617static void perf_event_sync_stat(struct perf_event_context *ctx,
 3618				   struct perf_event_context *next_ctx)
 3619{
 3620	struct perf_event *event, *next_event;
 3621
 3622	if (!ctx->nr_stat)
 3623		return;
 3624
 3625	update_context_time(ctx);
 3626
 3627	event = list_first_entry(&ctx->event_list,
 3628				   struct perf_event, event_entry);
 3629
 3630	next_event = list_first_entry(&next_ctx->event_list,
 3631					struct perf_event, event_entry);
 3632
 3633	while (&event->event_entry != &ctx->event_list &&
 3634	       &next_event->event_entry != &next_ctx->event_list) {
 3635
 3636		__perf_event_sync_stat(event, next_event);
 3637
 3638		event = list_next_entry(event, event_entry);
 3639		next_event = list_next_entry(next_event, event_entry);
 3640	}
 3641}
 3642
 3643static void perf_ctx_sched_task_cb(struct perf_event_context *ctx,
 3644				   struct task_struct *task, bool sched_in)
 3645{
 3646	struct perf_event_pmu_context *pmu_ctx;
 3647	struct perf_cpu_pmu_context *cpc;
 3648
 3649	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
 3650		cpc = this_cpc(pmu_ctx->pmu);
 3651
 3652		if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task)
 3653			pmu_ctx->pmu->sched_task(pmu_ctx, task, sched_in);
 3654	}
 3655}
 3656
 3657static void
 3658perf_event_context_sched_out(struct task_struct *task, struct task_struct *next)
 3659{
 3660	struct perf_event_context *ctx = task->perf_event_ctxp;
 3661	struct perf_event_context *next_ctx;
 3662	struct perf_event_context *parent, *next_parent;
 3663	int do_switch = 1;
 3664
 3665	if (likely(!ctx))
 3666		return;
 3667
 3668	rcu_read_lock();
 3669	next_ctx = rcu_dereference(next->perf_event_ctxp);
 3670	if (!next_ctx)
 3671		goto unlock;
 3672
 3673	parent = rcu_dereference(ctx->parent_ctx);
 3674	next_parent = rcu_dereference(next_ctx->parent_ctx);
 3675
 3676	/* If neither context have a parent context; they cannot be clones. */
 3677	if (!parent && !next_parent)
 3678		goto unlock;
 3679
 3680	if (next_parent == ctx || next_ctx == parent || next_parent == parent) {
 3681		/*
 3682		 * Looks like the two contexts are clones, so we might be
 3683		 * able to optimize the context switch.  We lock both
 3684		 * contexts and check that they are clones under the
 3685		 * lock (including re-checking that neither has been
 3686		 * uncloned in the meantime).  It doesn't matter which
 3687		 * order we take the locks because no other cpu could
 3688		 * be trying to lock both of these tasks.
 3689		 */
 3690		raw_spin_lock(&ctx->lock);
 3691		raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
 3692		if (context_equiv(ctx, next_ctx)) {
 3693
 3694			perf_ctx_disable(ctx, false);
 3695
 3696			/* PMIs are disabled; ctx->nr_no_switch_fast is stable. */
 3697			if (local_read(&ctx->nr_no_switch_fast) ||
 3698			    local_read(&next_ctx->nr_no_switch_fast)) {
 3699				/*
 3700				 * Must not swap out ctx when there's pending
 3701				 * events that rely on the ctx->task relation.
 3702				 *
 3703				 * Likewise, when a context contains inherit +
 3704				 * SAMPLE_READ events they should be switched
 3705				 * out using the slow path so that they are
 3706				 * treated as if they were distinct contexts.
 3707				 */
 3708				raw_spin_unlock(&next_ctx->lock);
 3709				rcu_read_unlock();
 3710				goto inside_switch;
 3711			}
 3712
 3713			WRITE_ONCE(ctx->task, next);
 3714			WRITE_ONCE(next_ctx->task, task);
 3715
 3716			perf_ctx_sched_task_cb(ctx, task, false);
 3717
 3718			perf_ctx_enable(ctx, false);
 3719
 3720			/*
 3721			 * RCU_INIT_POINTER here is safe because we've not
 3722			 * modified the ctx and the above modification of
 3723			 * ctx->task is immaterial since this value is
 3724			 * always verified under ctx->lock which we're now
 3725			 * holding.
 3726			 */
 3727			RCU_INIT_POINTER(task->perf_event_ctxp, next_ctx);
 3728			RCU_INIT_POINTER(next->perf_event_ctxp, ctx);
 3729
 3730			do_switch = 0;
 3731
 3732			perf_event_sync_stat(ctx, next_ctx);
 3733		}
 3734		raw_spin_unlock(&next_ctx->lock);
 3735		raw_spin_unlock(&ctx->lock);
 3736	}
 3737unlock:
 3738	rcu_read_unlock();
 3739
 3740	if (do_switch) {
 3741		raw_spin_lock(&ctx->lock);
 3742		perf_ctx_disable(ctx, false);
 3743
 3744inside_switch:
 3745		perf_ctx_sched_task_cb(ctx, task, false);
 3746		task_ctx_sched_out(ctx, NULL, EVENT_ALL);
 3747
 3748		perf_ctx_enable(ctx, false);
 3749		raw_spin_unlock(&ctx->lock);
 3750	}
 3751}
 3752
 3753static DEFINE_PER_CPU(struct list_head, sched_cb_list);
 3754static DEFINE_PER_CPU(int, perf_sched_cb_usages);
 3755
 3756void perf_sched_cb_dec(struct pmu *pmu)
 3757{
 3758	struct perf_cpu_pmu_context *cpc = this_cpc(pmu);
 3759
 3760	this_cpu_dec(perf_sched_cb_usages);
 3761	barrier();
 3762
 3763	if (!--cpc->sched_cb_usage)
 3764		list_del(&cpc->sched_cb_entry);
 3765}
 3766
 3767
 3768void perf_sched_cb_inc(struct pmu *pmu)
 3769{
 3770	struct perf_cpu_pmu_context *cpc = this_cpc(pmu);
 3771
 3772	if (!cpc->sched_cb_usage++)
 3773		list_add(&cpc->sched_cb_entry, this_cpu_ptr(&sched_cb_list));
 3774
 3775	barrier();
 3776	this_cpu_inc(perf_sched_cb_usages);
 3777}
 3778
 3779/*
 3780 * This function provides the context switch callback to the lower code
 3781 * layer. It is invoked ONLY when the context switch callback is enabled.
 3782 *
 3783 * This callback is relevant even to per-cpu events; for example multi event
 3784 * PEBS requires this to provide PID/TID information. This requires we flush
 3785 * all queued PEBS records before we context switch to a new task.
 3786 */
 3787static void __perf_pmu_sched_task(struct perf_cpu_pmu_context *cpc,
 3788				  struct task_struct *task, bool sched_in)
 3789{
 3790	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
 3791	struct pmu *pmu;
 3792
 3793	pmu = cpc->epc.pmu;
 3794
 3795	/* software PMUs will not have sched_task */
 3796	if (WARN_ON_ONCE(!pmu->sched_task))
 3797		return;
 3798
 3799	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
 3800	perf_pmu_disable(pmu);
 3801
 3802	pmu->sched_task(cpc->task_epc, task, sched_in);
 3803
 3804	perf_pmu_enable(pmu);
 3805	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
 3806}
 3807
 3808static void perf_pmu_sched_task(struct task_struct *prev,
 3809				struct task_struct *next,
 3810				bool sched_in)
 3811{
 3812	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
 3813	struct perf_cpu_pmu_context *cpc;
 3814
 3815	/* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */
 3816	if (prev == next || cpuctx->task_ctx)
 3817		return;
 3818
 3819	list_for_each_entry(cpc, this_cpu_ptr(&sched_cb_list), sched_cb_entry)
 3820		__perf_pmu_sched_task(cpc, sched_in ? next : prev, sched_in);
 3821}
 3822
 3823static void perf_event_switch(struct task_struct *task,
 3824			      struct task_struct *next_prev, bool sched_in);
 3825
 3826/*
 3827 * Called from scheduler to remove the events of the current task,
 3828 * with interrupts disabled.
 3829 *
 3830 * We stop each event and update the event value in event->count.
 3831 *
 3832 * This does not protect us against NMI, but disable()
 3833 * sets the disabled bit in the control field of event _before_
 3834 * accessing the event control register. If a NMI hits, then it will
 3835 * not restart the event.
 3836 */
 3837void __perf_event_task_sched_out(struct task_struct *task,
 3838				 struct task_struct *next)
 3839{
 3840	if (__this_cpu_read(perf_sched_cb_usages))
 3841		perf_pmu_sched_task(task, next, false);
 3842
 3843	if (atomic_read(&nr_switch_events))
 3844		perf_event_switch(task, next, false);
 3845
 3846	perf_event_context_sched_out(task, next);
 3847
 3848	/*
 3849	 * if cgroup events exist on this CPU, then we need
 3850	 * to check if we have to switch out PMU state.
 3851	 * cgroup event are system-wide mode only
 3852	 */
 3853	perf_cgroup_switch(next);
 3854}
 3855
 3856static bool perf_less_group_idx(const void *l, const void *r, void __always_unused *args)
 3857{
 3858	const struct perf_event *le = *(const struct perf_event **)l;
 3859	const struct perf_event *re = *(const struct perf_event **)r;
 3860
 3861	return le->group_index < re->group_index;
 3862}
 3863
 3864DEFINE_MIN_HEAP(struct perf_event *, perf_event_min_heap);
 3865
 3866static const struct min_heap_callbacks perf_min_heap = {
 3867	.less = perf_less_group_idx,
 3868	.swp = NULL,
 3869};
 3870
 3871static void __heap_add(struct perf_event_min_heap *heap, struct perf_event *event)
 3872{
 3873	struct perf_event **itrs = heap->data;
 3874
 3875	if (event) {
 3876		itrs[heap->nr] = event;
 3877		heap->nr++;
 3878	}
 3879}
 3880
 3881static void __link_epc(struct perf_event_pmu_context *pmu_ctx)
 3882{
 3883	struct perf_cpu_pmu_context *cpc;
 3884
 3885	if (!pmu_ctx->ctx->task)
 3886		return;
 3887
 3888	cpc = this_cpc(pmu_ctx->pmu);
 3889	WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);
 3890	cpc->task_epc = pmu_ctx;
 3891}
 3892
 3893static noinline int visit_groups_merge(struct perf_event_context *ctx,
 3894				struct perf_event_groups *groups, int cpu,
 3895				struct pmu *pmu,
 3896				int (*func)(struct perf_event *, void *),
 3897				void *data)
 3898{
 3899#ifdef CONFIG_CGROUP_PERF
 3900	struct cgroup_subsys_state *css = NULL;
 3901#endif
 3902	struct perf_cpu_context *cpuctx = NULL;
 3903	/* Space for per CPU and/or any CPU event iterators. */
 3904	struct perf_event *itrs[2];
 3905	struct perf_event_min_heap event_heap;
 3906	struct perf_event **evt;
 3907	int ret;
 3908
 3909	if (pmu->filter && pmu->filter(pmu, cpu))
 3910		return 0;
 3911
 3912	if (!ctx->task) {
 3913		cpuctx = this_cpu_ptr(&perf_cpu_context);
 3914		event_heap = (struct perf_event_min_heap){
 3915			.data = cpuctx->heap,
 3916			.nr = 0,
 3917			.size = cpuctx->heap_size,
 3918		};
 3919
 3920		lockdep_assert_held(&cpuctx->ctx.lock);
 3921
 3922#ifdef CONFIG_CGROUP_PERF
 3923		if (cpuctx->cgrp)
 3924			css = &cpuctx->cgrp->css;
 3925#endif
 3926	} else {
 3927		event_heap = (struct perf_event_min_heap){
 3928			.data = itrs,
 3929			.nr = 0,
 3930			.size = ARRAY_SIZE(itrs),
 3931		};
 3932		/* Events not within a CPU context may be on any CPU. */
 3933		__heap_add(&event_heap, perf_event_groups_first(groups, -1, pmu, NULL));
 3934	}
 3935	evt = event_heap.data;
 3936
 3937	__heap_add(&event_heap, perf_event_groups_first(groups, cpu, pmu, NULL));
 3938
 3939#ifdef CONFIG_CGROUP_PERF
 3940	for (; css; css = css->parent)
 3941		__heap_add(&event_heap, perf_event_groups_first(groups, cpu, pmu, css->cgroup));
 3942#endif
 3943
 3944	if (event_heap.nr) {
 3945		__link_epc((*evt)->pmu_ctx);
 3946		perf_assert_pmu_disabled((*evt)->pmu_ctx->pmu);
 3947	}
 3948
 3949	min_heapify_all_inline(&event_heap, &perf_min_heap, NULL);
 3950
 3951	while (event_heap.nr) {
 3952		ret = func(*evt, data);
 3953		if (ret)
 3954			return ret;
 3955
 3956		*evt = perf_event_groups_next(*evt, pmu);
 3957		if (*evt)
 3958			min_heap_sift_down_inline(&event_heap, 0, &perf_min_heap, NULL);
 3959		else
 3960			min_heap_pop_inline(&event_heap, &perf_min_heap, NULL);
 3961	}
 3962
 3963	return 0;
 3964}
 3965
 3966/*
 3967 * Because the userpage is strictly per-event (there is no concept of context,
 3968 * so there cannot be a context indirection), every userpage must be updated
 3969 * when context time starts :-(
 3970 *
 3971 * IOW, we must not miss EVENT_TIME edges.
 3972 */
 3973static inline bool event_update_userpage(struct perf_event *event)
 3974{
 3975	if (likely(!refcount_read(&event->mmap_count)))
 3976		return false;
 3977
 3978	perf_event_update_time(event);
 3979	perf_event_update_userpage(event);
 3980
 3981	return true;
 3982}
 3983
 3984static inline void group_update_userpage(struct perf_event *group_event)
 3985{
 3986	struct perf_event *event;
 3987
 3988	if (!event_update_userpage(group_event))
 3989		return;
 3990
 3991	for_each_sibling_event(event, group_event)
 3992		event_update_userpage(event);
 3993}
 3994
 3995static int merge_sched_in(struct perf_event *event, void *data)
 3996{
 3997	struct perf_event_context *ctx = event->ctx;
 3998	int *can_add_hw = data;
 3999
 4000	if (event->state <= PERF_EVENT_STATE_OFF)
 4001		return 0;
 4002
 4003	if (!event_filter_match(event))
 4004		return 0;
 4005
 4006	if (group_can_go_on(event, *can_add_hw)) {
 4007		if (!group_sched_in(event, ctx))
 4008			list_add_tail(&event->active_list, get_event_list(event));
 4009	}
 4010
 4011	if (event->state == PERF_EVENT_STATE_INACTIVE) {
 4012		*can_add_hw = 0;
 4013		if (event->attr.pinned) {
 4014			perf_cgroup_event_disable(event, ctx);
 4015			perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
 4016
 4017			if (*perf_event_fasync(event))
 4018				event->pending_kill = POLL_ERR;
 4019
 4020			event->pending_wakeup = 1;
 4021			irq_work_queue(&event->pending_irq);
 4022		} else {
 4023			struct perf_cpu_pmu_context *cpc = this_cpc(event->pmu_ctx->pmu);
 4024
 4025			event->pmu_ctx->rotate_necessary = 1;
 4026			perf_mux_hrtimer_restart(cpc);
 4027			group_update_userpage(event);
 4028		}
 4029	}
 4030
 4031	return 0;
 4032}
 4033
 4034static void pmu_groups_sched_in(struct perf_event_context *ctx,
 4035				struct perf_event_groups *groups,
 4036				struct pmu *pmu)
 4037{
 4038	int can_add_hw = 1;
 4039	visit_groups_merge(ctx, groups, smp_processor_id(), pmu,
 4040			   merge_sched_in, &can_add_hw);
 4041}
 4042
 4043static void __pmu_ctx_sched_in(struct perf_event_pmu_context *pmu_ctx,
 4044			       enum event_type_t event_type)
 4045{
 4046	struct perf_event_context *ctx = pmu_ctx->ctx;
 4047
 4048	if (event_type & EVENT_PINNED)
 4049		pmu_groups_sched_in(ctx, &ctx->pinned_groups, pmu_ctx->pmu);
 4050	if (event_type & EVENT_FLEXIBLE)
 4051		pmu_groups_sched_in(ctx, &ctx->flexible_groups, pmu_ctx->pmu);
 4052}
 4053
 4054static void
 4055ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type)
 4056{
 4057	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
 4058	struct perf_event_pmu_context *pmu_ctx;
 4059	int is_active = ctx->is_active;
 4060	bool cgroup = event_type & EVENT_CGROUP;
 4061
 4062	event_type &= ~EVENT_CGROUP;
 4063
 4064	lockdep_assert_held(&ctx->lock);
 4065
 4066	if (likely(!ctx->nr_events))
 4067		return;
 4068
 4069	if (!(is_active & EVENT_TIME)) {
 4070		/* start ctx time */
 4071		__update_context_time(ctx, false);
 4072		perf_cgroup_set_timestamp(cpuctx);
 4073		/*
 4074		 * CPU-release for the below ->is_active store,
 4075		 * see __load_acquire() in perf_event_time_now()
 4076		 */
 4077		barrier();
 4078	}
 4079
 4080	ctx->is_active |= (event_type | EVENT_TIME);
 4081	if (ctx->task) {
 4082		if (!(is_active & EVENT_ALL))
 4083			cpuctx->task_ctx = ctx;
 4084		else
 4085			WARN_ON_ONCE(cpuctx->task_ctx != ctx);
 4086	}
 4087
 4088	is_active ^= ctx->is_active; /* changed bits */
 4089
 4090	/*
 4091	 * First go through the list and put on any pinned groups
 4092	 * in order to give them the best chance of going on.
 4093	 */
 4094	if (is_active & EVENT_PINNED) {
 4095		for_each_epc(pmu_ctx, ctx, pmu, cgroup)
 4096			__pmu_ctx_sched_in(pmu_ctx, EVENT_PINNED);
 4097	}
 4098
 4099	/* Then walk through the lower prio flexible groups */
 4100	if (is_active & EVENT_FLEXIBLE) {
 4101		for_each_epc(pmu_ctx, ctx, pmu, cgroup)
 4102			__pmu_ctx_sched_in(pmu_ctx, EVENT_FLEXIBLE);
 4103	}
 4104}
 4105
 4106static void perf_event_context_sched_in(struct task_struct *task)
 4107{
 4108	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
 4109	struct perf_event_context *ctx;
 4110
 4111	rcu_read_lock();
 4112	ctx = rcu_dereference(task->perf_event_ctxp);
 4113	if (!ctx)
 4114		goto rcu_unlock;
 4115
 4116	if (cpuctx->task_ctx == ctx) {
 4117		perf_ctx_lock(cpuctx, ctx);
 4118		perf_ctx_disable(ctx, false);
 4119
 4120		perf_ctx_sched_task_cb(ctx, task, true);
 4121
 4122		perf_ctx_enable(ctx, false);
 4123		perf_ctx_unlock(cpuctx, ctx);
 4124		goto rcu_unlock;
 4125	}
 4126
 4127	perf_ctx_lock(cpuctx, ctx);
 4128	/*
 4129	 * We must check ctx->nr_events while holding ctx->lock, such
 4130	 * that we serialize against perf_install_in_context().
 4131	 */
 4132	if (!ctx->nr_events)
 4133		goto unlock;
 4134
 4135	perf_ctx_disable(ctx, false);
 4136	/*
 4137	 * We want to keep the following priority order:
 4138	 * cpu pinned (that don't need to move), task pinned,
 4139	 * cpu flexible, task flexible.
 4140	 *
 4141	 * However, if task's ctx is not carrying any pinned
 4142	 * events, no need to flip the cpuctx's events around.
 4143	 */
 4144	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) {
 4145		perf_ctx_disable(&cpuctx->ctx, false);
 4146		ctx_sched_out(&cpuctx->ctx, NULL, EVENT_FLEXIBLE);
 4147	}
 4148
 4149	perf_event_sched_in(cpuctx, ctx, NULL);
 4150
 4151	perf_ctx_sched_task_cb(cpuctx->task_ctx, task, true);
 4152
 4153	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))
 4154		perf_ctx_enable(&cpuctx->ctx, false);
 4155
 4156	perf_ctx_enable(ctx, false);
 4157
 4158unlock:
 4159	perf_ctx_unlock(cpuctx, ctx);
 4160rcu_unlock:
 4161	rcu_read_unlock();
 4162}
 4163
 4164/*
 4165 * Called from scheduler to add the events of the current task
 4166 * with interrupts disabled.
 4167 *
 4168 * We restore the event value and then enable it.
 4169 *
 4170 * This does not protect us against NMI, but enable()
 4171 * sets the enabled bit in the control field of event _before_
 4172 * accessing the event control register. If a NMI hits, then it will
 4173 * keep the event running.
 4174 */
 4175void __perf_event_task_sched_in(struct task_struct *prev,
 4176				struct task_struct *task)
 4177{
 4178	perf_event_context_sched_in(task);
 4179
 4180	if (atomic_read(&nr_switch_events))
 4181		perf_event_switch(task, prev, true);
 4182
 4183	if (__this_cpu_read(perf_sched_cb_usages))
 4184		perf_pmu_sched_task(prev, task, true);
 4185}
 4186
 4187static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
 4188{
 4189	u64 frequency = event->attr.sample_freq;
 4190	u64 sec = NSEC_PER_SEC;
 4191	u64 divisor, dividend;
 4192
 4193	int count_fls, nsec_fls, frequency_fls, sec_fls;
 4194
 4195	count_fls = fls64(count);
 4196	nsec_fls = fls64(nsec);
 4197	frequency_fls = fls64(frequency);
 4198	sec_fls = 30;
 4199
 4200	/*
 4201	 * We got @count in @nsec, with a target of sample_freq HZ
 4202	 * the target period becomes:
 4203	 *
 4204	 *             @count * 10^9
 4205	 * period = -------------------
 4206	 *          @nsec * sample_freq
 4207	 *
 4208	 */
 4209
 4210	/*
 4211	 * Reduce accuracy by one bit such that @a and @b converge
 4212	 * to a similar magnitude.
 4213	 */
 4214#define REDUCE_FLS(a, b)		\
 4215do {					\
 4216	if (a##_fls > b##_fls) {	\
 4217		a >>= 1;		\
 4218		a##_fls--;		\
 4219	} else {			\
 4220		b >>= 1;		\
 4221		b##_fls--;		\
 4222	}				\
 4223} while (0)
 4224
 4225	/*
 4226	 * Reduce accuracy until either term fits in a u64, then proceed with
 4227	 * the other, so that finally we can do a u64/u64 division.
 4228	 */
 4229	while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
 4230		REDUCE_FLS(nsec, frequency);
 4231		REDUCE_FLS(sec, count);
 4232	}
 4233
 4234	if (count_fls + sec_fls > 64) {
 4235		divisor = nsec * frequency;
 4236
 4237		while (count_fls + sec_fls > 64) {
 4238			REDUCE_FLS(count, sec);
 4239			divisor >>= 1;
 4240		}
 4241
 4242		dividend = count * sec;
 4243	} else {
 4244		dividend = count * sec;
 4245
 4246		while (nsec_fls + frequency_fls > 64) {
 4247			REDUCE_FLS(nsec, frequency);
 4248			dividend >>= 1;
 4249		}
 4250
 4251		divisor = nsec * frequency;
 4252	}
 4253
 4254	if (!divisor)
 4255		return dividend;
 4256
 4257	return div64_u64(dividend, divisor);
 4258}
 4259
 4260static DEFINE_PER_CPU(int, perf_throttled_count);
 4261static DEFINE_PER_CPU(u64, perf_throttled_seq);
 4262
 4263static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
 4264{
 4265	struct hw_perf_event *hwc = &event->hw;
 4266	s64 period, sample_period;
 4267	s64 delta;
 4268
 4269	period = perf_calculate_period(event, nsec, count);
 4270
 4271	delta = (s64)(period - hwc->sample_period);
 4272	if (delta >= 0)
 4273		delta += 7;
 4274	else
 4275		delta -= 7;
 4276	delta /= 8; /* low pass filter */
 4277
 4278	sample_period = hwc->sample_period + delta;
 4279
 4280	if (!sample_period)
 4281		sample_period = 1;
 4282
 4283	hwc->sample_period = sample_period;
 4284
 4285	if (local64_read(&hwc->period_left) > 8*sample_period) {
 4286		if (disable)
 4287			event->pmu->stop(event, PERF_EF_UPDATE);
 4288
 4289		local64_set(&hwc->period_left, 0);
 4290
 4291		if (disable)
 4292			event->pmu->start(event, PERF_EF_RELOAD);
 4293	}
 4294}
 4295
 4296static void perf_adjust_freq_unthr_events(struct list_head *event_list)
 4297{
 4298	struct perf_event *event;
 4299	struct hw_perf_event *hwc;
 4300	u64 now, period = TICK_NSEC;
 4301	s64 delta;
 4302
 4303	list_for_each_entry(event, event_list, active_list) {
 4304		if (event->state != PERF_EVENT_STATE_ACTIVE)
 4305			continue;
 4306
 4307		// XXX use visit thingy to avoid the -1,cpu match
 4308		if (!event_filter_match(event))
 4309			continue;
 4310
 4311		hwc = &event->hw;
 4312
 4313		if (hwc->interrupts == MAX_INTERRUPTS)
 4314			perf_event_unthrottle_group(event, is_event_in_freq_mode(event));
 4315
 4316		if (!is_event_in_freq_mode(event))
 4317			continue;
 4318
 4319		/*
 4320		 * stop the event and update event->count
 4321		 */
 4322		event->pmu->stop(event, PERF_EF_UPDATE);
 4323
 4324		now = local64_read(&event->count);
 4325		delta = now - hwc->freq_count_stamp;
 4326		hwc->freq_count_stamp = now;
 4327
 4328		/*
 4329		 * restart the event
 4330		 * reload only if value has changed
 4331		 * we have stopped the event so tell that
 4332		 * to perf_adjust_period() to avoid stopping it
 4333		 * twice.
 4334		 */
 4335		if (delta > 0)
 4336			perf_adjust_period(event, period, delta, false);
 4337
 4338		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
 4339	}
 4340}
 4341
 4342/*
 4343 * combine freq adjustment with unthrottling to avoid two passes over the
 4344 * events. At the same time, make sure, having freq events does not change
 4345 * the rate of unthrottling as that would introduce bias.
 4346 */
 4347static void
 4348perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
 4349{
 4350	struct perf_event_pmu_context *pmu_ctx;
 4351
 4352	/*
 4353	 * only need to iterate over all events iff:
 4354	 * - context have events in frequency mode (needs freq adjust)
 4355	 * - there are events to unthrottle on this cpu
 4356	 */
 4357	if (!(ctx->nr_freq || unthrottle))
 4358		return;
 4359
 4360	raw_spin_lock(&ctx->lock);
 4361
 4362	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
 4363		if (!(pmu_ctx->nr_freq || unthrottle))
 4364			continue;
 4365		if (!perf_pmu_ctx_is_active(pmu_ctx))
 4366			continue;
 4367		if (pmu_ctx->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT)
 4368			continue;
 4369
 4370		perf_pmu_disable(pmu_ctx->pmu);
 4371		perf_adjust_freq_unthr_events(&pmu_ctx->pinned_active);
 4372		perf_adjust_freq_unthr_events(&pmu_ctx->flexible_active);
 4373		perf_pmu_enable(pmu_ctx->pmu);
 4374	}
 4375
 4376	raw_spin_unlock(&ctx->lock);
 4377}
 4378
 4379/*
 4380 * Move @event to the tail of the @ctx's elegible events.
 4381 */
 4382static void rotate_ctx(struct perf_event_context *ctx, struct perf_event *event)
 4383{
 4384	/*
 4385	 * Rotate the first entry last of non-pinned groups. Rotation might be
 4386	 * disabled by the inheritance code.
 4387	 */
 4388	if (ctx->rotate_disable)
 4389		return;
 4390
 4391	perf_event_groups_delete(&ctx->flexible_groups, event);
 4392	perf_event_groups_insert(&ctx->flexible_groups, event);
 4393}
 4394
 4395/* pick an event from the flexible_groups to rotate */
 4396static inline struct perf_event *
 4397ctx_event_to_rotate(struct perf_event_pmu_context *pmu_ctx)
 4398{
 4399	struct perf_event *event;
 4400	struct rb_node *node;
 4401	struct rb_root *tree;
 4402	struct __group_key key = {
 4403		.pmu = pmu_ctx->pmu,
 4404	};
 4405
 4406	/* pick the first active flexible event */
 4407	event = list_first_entry_or_null(&pmu_ctx->flexible_active,
 4408					 struct perf_event, active_list);
 4409	if (event)
 4410		goto out;
 4411
 4412	/* if no active flexible event, pick the first event */
 4413	tree = &pmu_ctx->ctx->flexible_groups.tree;
 4414
 4415	if (!pmu_ctx->ctx->task) {
 4416		key.cpu = smp_processor_id();
 4417
 4418		node = rb_find_first(&key, tree, __group_cmp_ignore_cgroup);
 4419		if (node)
 4420			event = __node_2_pe(node);
 4421		goto out;
 4422	}
 4423
 4424	key.cpu = -1;
 4425	node = rb_find_first(&key, tree, __group_cmp_ignore_cgroup);
 4426	if (node) {
 4427		event = __node_2_pe(node);
 4428		goto out;
 4429	}
 4430
 4431	key.cpu = smp_processor_id();
 4432	node = rb_find_first(&key, tree, __group_cmp_ignore_cgroup);
 4433	if (node)
 4434		event = __node_2_pe(node);
 4435
 4436out:
 4437	/*
 4438	 * Unconditionally clear rotate_necessary; if ctx_flexible_sched_in()
 4439	 * finds there are unschedulable events, it will set it again.
 4440	 */
 4441	pmu_ctx->rotate_necessary = 0;
 4442
 4443	return event;
 4444}
 4445
 4446static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc)
 4447{
 4448	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
 4449	struct perf_event_pmu_context *cpu_epc, *task_epc = NULL;
 4450	struct perf_event *cpu_event = NULL, *task_event = NULL;
 4451	int cpu_rotate, task_rotate;
 4452	struct pmu *pmu;
 4453
 4454	/*
 4455	 * Since we run this from IRQ context, nobody can install new
 4456	 * events, thus the event count values are stable.
 4457	 */
 4458
 4459	cpu_epc = &cpc->epc;
 4460	pmu = cpu_epc->pmu;
 4461	task_epc = cpc->task_epc;
 4462
 4463	cpu_rotate = cpu_epc->rotate_necessary;
 4464	task_rotate = task_epc ? task_epc->rotate_necessary : 0;
 4465
 4466	if (!(cpu_rotate || task_rotate))
 4467		return false;
 4468
 4469	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
 4470	perf_pmu_disable(pmu);
 4471
 4472	if (task_rotate)
 4473		task_event = ctx_event_to_rotate(task_epc);
 4474	if (cpu_rotate)
 4475		cpu_event = ctx_event_to_rotate(cpu_epc);
 4476
 4477	/*
 4478	 * As per the order given at ctx_resched() first 'pop' task flexible
 4479	 * and then, if needed CPU flexible.
 4480	 */
 4481	if (task_event || (task_epc && cpu_event)) {
 4482		update_context_time(task_epc->ctx);
 4483		__pmu_ctx_sched_out(task_epc, EVENT_FLEXIBLE);
 4484	}
 4485
 4486	if (cpu_event) {
 4487		update_context_time(&cpuctx->ctx);
 4488		__pmu_ctx_sched_out(cpu_epc, EVENT_FLEXIBLE);
 4489		rotate_ctx(&cpuctx->ctx, cpu_event);
 4490		__pmu_ctx_sched_in(cpu_epc, EVENT_FLEXIBLE);
 4491	}
 4492
 4493	if (task_event)
 4494		rotate_ctx(task_epc->ctx, task_event);
 4495
 4496	if (task_event || (task_epc && cpu_event))
 4497		__pmu_ctx_sched_in(task_epc, EVENT_FLEXIBLE);
 4498
 4499	perf_pmu_enable(pmu);
 4500	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
 4501
 4502	return true;
 4503}
 4504
 4505void perf_event_task_tick(void)
 4506{
 4507	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
 4508	struct perf_event_context *ctx;
 4509	int throttled;
 4510
 4511	lockdep_assert_irqs_disabled();
 4512
 4513	__this_cpu_inc(perf_throttled_seq);
 4514	throttled = __this_cpu_xchg(perf_throttled_count, 0);
 4515	tick_dep_clear_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
 4516
 4517	perf_adjust_freq_unthr_context(&cpuctx->ctx, !!throttled);
 4518
 4519	rcu_read_lock();
 4520	ctx = rcu_dereference(current->perf_event_ctxp);
 4521	if (ctx)
 4522		perf_adjust_freq_unthr_context(ctx, !!throttled);
 4523	rcu_read_unlock();
 4524}
 4525
 4526static int event_enable_on_exec(struct perf_event *event,
 4527				struct perf_event_context *ctx)
 4528{
 4529	if (!event->attr.enable_on_exec)
 4530		return 0;
 4531
 4532	event->attr.enable_on_exec = 0;
 4533	if (event->state >= PERF_EVENT_STATE_INACTIVE)
 4534		return 0;
 4535
 4536	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
 4537
 4538	return 1;
 4539}
 4540
 4541/*
 4542 * Enable all of a task's events that have been marked enable-on-exec.
 4543 * This expects task == current.
 4544 */
 4545static void perf_event_enable_on_exec(struct perf_event_context *ctx)
 4546{
 4547	struct perf_event_context *clone_ctx = NULL;
 4548	enum event_type_t event_type = 0;
 4549	struct perf_cpu_context *cpuctx;
 4550	struct perf_event *event;
 4551	unsigned long flags;
 4552	int enabled = 0;
 4553
 4554	local_irq_save(flags);
 4555	if (WARN_ON_ONCE(current->perf_event_ctxp != ctx))
 4556		goto out;
 4557
 4558	if (!ctx->nr_events)
 4559		goto out;
 4560
 4561	cpuctx = this_cpu_ptr(&perf_cpu_context);
 4562	perf_ctx_lock(cpuctx, ctx);
 4563	ctx_time_freeze(cpuctx, ctx);
 4564
 4565	list_for_each_entry(event, &ctx->event_list, event_entry) {
 4566		enabled |= event_enable_on_exec(event, ctx);
 4567		event_type |= get_event_type(event);
 4568	}
 4569
 4570	/*
 4571	 * Unclone and reschedule this context if we enabled any event.
 4572	 */
 4573	if (enabled) {
 4574		clone_ctx = unclone_ctx(ctx);
 4575		ctx_resched(cpuctx, ctx, NULL, event_type);
 4576	}
 4577	perf_ctx_unlock(cpuctx, ctx);
 4578
 4579out:
 4580	local_irq_restore(flags);
 4581
 4582	if (clone_ctx)
 4583		put_ctx(clone_ctx);
 4584}
 4585
 4586static void perf_remove_from_owner(struct perf_event *event);
 4587static void perf_event_exit_event(struct perf_event *event,
 4588				  struct perf_event_context *ctx,
 4589				  struct task_struct *task,
 4590				  bool revoke);
 4591
 4592/*
 4593 * Removes all events from the current task that have been marked
 4594 * remove-on-exec, and feeds their values back to parent events.
 4595 */
 4596static void perf_event_remove_on_exec(struct perf_event_context *ctx)
 4597{
 4598	struct perf_event_context *clone_ctx = NULL;
 4599	struct perf_event *event, *next;
 4600	unsigned long flags;
 4601	bool modified = false;
 4602
 4603	mutex_lock(&ctx->mutex);
 4604
 4605	if (WARN_ON_ONCE(ctx->task != current))
 4606		goto unlock;
 4607
 4608	list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) {
 4609		if (!event->attr.remove_on_exec)
 4610			continue;
 4611
 4612		if (!is_kernel_event(event))
 4613			perf_remove_from_owner(event);
 4614
 4615		modified = true;
 4616
 4617		perf_event_exit_event(event, ctx, ctx->task, false);
 4618	}
 4619
 4620	raw_spin_lock_irqsave(&ctx->lock, flags);
 4621	if (modified)
 4622		clone_ctx = unclone_ctx(ctx);
 4623	raw_spin_unlock_irqrestore(&ctx->lock, flags);
 4624
 4625unlock:
 4626	mutex_unlock(&ctx->mutex);
 4627
 4628	if (clone_ctx)
 4629		put_ctx(clone_ctx);
 4630}
 4631
 4632struct perf_read_data {
 4633	struct perf_event *event;
 4634	bool group;
 4635	int ret;
 4636};
 4637
 4638static inline const struct cpumask *perf_scope_cpu_topology_cpumask(unsigned int scope, int cpu);
 4639
 4640static int __perf_event_read_cpu(struct perf_event *event, int event_cpu)
 4641{
 4642	int local_cpu = smp_processor_id();
 4643	u16 local_pkg, event_pkg;
 4644
 4645	if ((unsigned)event_cpu >= nr_cpu_ids)
 4646		return event_cpu;
 4647
 4648	if (event->group_caps & PERF_EV_CAP_READ_SCOPE) {
 4649		const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(event->pmu->scope, event_cpu);
 4650
 4651		if (cpumask && cpumask_test_cpu(local_cpu, cpumask))
 4652			return local_cpu;
 4653	}
 4654
 4655	if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) {
 4656		event_pkg = topology_physical_package_id(event_cpu);
 4657		local_pkg = topology_physical_package_id(local_cpu);
 4658
 4659		if (event_pkg == local_pkg)
 4660			return local_cpu;
 4661	}
 4662
 4663	return event_cpu;
 4664}
 4665
 4666/*
 4667 * Cross CPU call to read the hardware event
 4668 */
 4669static void __perf_event_read(void *info)
 4670{
 4671	struct perf_read_data *data = info;
 4672	struct perf_event *sub, *event = data->event;
 4673	struct perf_event_context *ctx = event->ctx;
 4674	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
 4675	struct pmu *pmu;
 4676
 4677	/*
 4678	 * If this is a task context, we need to check whether it is
 4679	 * the current task context of this cpu.  If not it has been
 4680	 * scheduled out before the smp call arrived.  In that case
 4681	 * event->count would have been updated to a recent sample
 4682	 * when the event was scheduled out.
 4683	 */
 4684	if (ctx->task && cpuctx->task_ctx != ctx)
 4685		return;
 4686
 4687	guard(raw_spinlock)(&ctx->lock);
 4688	ctx_time_update_event(ctx, event);
 4689
 4690	perf_event_update_time(event);
 4691	if (data->group)
 4692		perf_event_update_sibling_time(event);
 4693
 4694	if (event->state != PERF_EVENT_STATE_ACTIVE)
 4695		return;
 4696
 4697	if (!data->group) {
 4698		perf_pmu_read(event);
 4699		data->ret = 0;
 4700		return;
 4701	}
 4702
 4703	pmu = event->pmu_ctx->pmu;
 4704	pmu->start_txn(pmu, PERF_PMU_TXN_READ);
 4705
 4706	perf_pmu_read(event);
 4707	for_each_sibling_event(sub, event)
 4708		perf_pmu_read(sub);
 4709
 4710	data->ret = pmu->commit_txn(pmu);
 4711}
 4712
 4713static inline u64 perf_event_count(struct perf_event *event, bool self)
 4714{
 4715	if (self)
 4716		return local64_read(&event->count);
 4717
 4718	return local64_read(&event->count) + atomic64_read(&event->child_count);
 4719}
 4720
 4721static void calc_timer_values(struct perf_event *event,
 4722				u64 *now,
 4723				u64 *enabled,
 4724				u64 *running)
 4725{
 4726	u64 ctx_time;
 4727
 4728	*now = perf_clock();
 4729	ctx_time = perf_event_time_now(event, *now);
 4730	__perf_update_times(event, ctx_time, enabled, running);
 4731}
 4732
 4733/*
 4734 * NMI-safe method to read a local event, that is an event that
 4735 * is:
 4736 *   - either for the current task, or for this CPU
 4737 *   - does not have inherit set, for inherited task events
 4738 *     will not be local and we cannot read them atomically
 4739 *   - must not have a pmu::count method
 4740 */
 4741int perf_event_read_local(struct perf_event *event, u64 *value,
 4742			  u64 *enabled, u64 *running)
 4743{
 4744	unsigned long flags;
 4745	int event_oncpu;
 4746	int event_cpu;
 4747	int ret = 0;
 4748
 4749	/*
 4750	 * Disabling interrupts avoids all counter scheduling (context
 4751	 * switches, timer based rotation and IPIs).
 4752	 */
 4753	local_irq_save(flags);
 4754
 4755	/*
 4756	 * It must not be an event with inherit set, we cannot read
 4757	 * all child counters from atomic context.
 4758	 */
 4759	if (event->attr.inherit) {
 4760		ret = -EOPNOTSUPP;
 4761		goto out;
 4762	}
 4763
 4764	/* If this is a per-task event, it must be for current */
 4765	if ((event->attach_state & PERF_ATTACH_TASK) &&
 4766	    event->hw.target != current) {
 4767		ret = -EINVAL;
 4768		goto out;
 4769	}
 4770
 4771	/*
 4772	 * Get the event CPU numbers, and adjust them to local if the event is
 4773	 * a per-package event that can be read locally
 4774	 */
 4775	event_oncpu = __perf_event_read_cpu(event, event->oncpu);
 4776	event_cpu = __perf_event_read_cpu(event, event->cpu);
 4777
 4778	/* If this is a per-CPU event, it must be for this CPU */
 4779	if (!(event->attach_state & PERF_ATTACH_TASK) &&
 4780	    event_cpu != smp_processor_id()) {
 4781		ret = -EINVAL;
 4782		goto out;
 4783	}
 4784
 4785	/* If this is a pinned event it must be running on this CPU */
 4786	if (event->attr.pinned && event_oncpu != smp_processor_id()) {
 4787		ret = -EBUSY;
 4788		goto out;
 4789	}
 4790
 4791	/*
 4792	 * If the event is currently on this CPU, its either a per-task event,
 4793	 * or local to this CPU. Furthermore it means its ACTIVE (otherwise
 4794	 * oncpu == -1).
 4795	 */
 4796	if (event_oncpu == smp_processor_id())
 4797		event->pmu->read(event);
 4798
 4799	*value = local64_read(&event->count);
 4800	if (enabled || running) {
 4801		u64 __enabled, __running, __now;
 4802
 4803		calc_timer_values(event, &__now, &__enabled, &__running);
 4804		if (enabled)
 4805			*enabled = __enabled;
 4806		if (running)
 4807			*running = __running;
 4808	}
 4809out:
 4810	local_irq_restore(flags);
 4811
 4812	return ret;
 4813}
 4814
 4815static int perf_event_read(struct perf_event *event, bool group)
 4816{
 4817	enum perf_event_state state = READ_ONCE(event->state);
 4818	int event_cpu, ret = 0;
 4819
 4820	/*
 4821	 * If event is enabled and currently active on a CPU, update the
 4822	 * value in the event structure:
 4823	 */
 4824again:
 4825	if (state == PERF_EVENT_STATE_ACTIVE) {
 4826		struct perf_read_data data;
 4827
 4828		/*
 4829		 * Orders the ->state and ->oncpu loads such that if we see
 4830		 * ACTIVE we must also see the right ->oncpu.
 4831		 *
 4832		 * Matches the smp_wmb() from event_sched_in().
 4833		 */
 4834		smp_rmb();
 4835
 4836		event_cpu = READ_ONCE(event->oncpu);
 4837		if ((unsigned)event_cpu >= nr_cpu_ids)
 4838			return 0;
 4839
 4840		data = (struct perf_read_data){
 4841			.event = event,
 4842			.group = group,
 4843			.ret = 0,
 4844		};
 4845
 4846		preempt_disable();
 4847		event_cpu = __perf_event_read_cpu(event, event_cpu);
 4848
 4849		/*
 4850		 * Purposely ignore the smp_call_function_single() return
 4851		 * value.
 4852		 *
 4853		 * If event_cpu isn't a valid CPU it means the event got
 4854		 * scheduled out and that will have updated the event count.
 4855		 *
 4856		 * Therefore, either way, we'll have an up-to-date event count
 4857		 * after this.
 4858		 */
 4859		(void)smp_call_function_single(event_cpu, __perf_event_read, &data, 1);
 4860		preempt_enable();
 4861		ret = data.ret;
 4862
 4863	} else if (state == PERF_EVENT_STATE_INACTIVE) {
 4864		struct perf_event_context *ctx = event->ctx;
 4865		unsigned long flags;
 4866
 4867		raw_spin_lock_irqsave(&ctx->lock, flags);
 4868		state = event->state;
 4869		if (state != PERF_EVENT_STATE_INACTIVE) {
 4870			raw_spin_unlock_irqrestore(&ctx->lock, flags);
 4871			goto again;
 4872		}
 4873
 4874		/*
 4875		 * May read while context is not active (e.g., thread is
 4876		 * blocked), in that case we cannot update context time
 4877		 */
 4878		ctx_time_update_event(ctx, event);
 4879
 4880		perf_event_update_time(event);
 4881		if (group)
 4882			perf_event_update_sibling_time(event);
 4883		raw_spin_unlock_irqrestore(&ctx->lock, flags);
 4884	}
 4885
 4886	return ret;
 4887}
 4888
 4889/*
 4890 * Initialize the perf_event context in a task_struct:
 4891 */
 4892static void __perf_event_init_context(struct perf_event_context *ctx)
 4893{
 4894	raw_spin_lock_init(&ctx->lock);
 4895	mutex_init(&ctx->mutex);
 4896	INIT_LIST_HEAD(&ctx->pmu_ctx_list);
 4897	perf_event_groups_init(&ctx->pinned_groups);
 4898	perf_event_groups_init(&ctx->flexible_groups);
 4899	INIT_LIST_HEAD(&ctx->event_list);
 4900	refcount_set(&ctx->refcount, 1);
 4901}
 4902
 4903static void
 4904__perf_init_event_pmu_context(struct perf_event_pmu_context *epc, struct pmu *pmu)
 4905{
 4906	epc->pmu = pmu;
 4907	INIT_LIST_HEAD(&epc->pmu_ctx_entry);
 4908	INIT_LIST_HEAD(&epc->pinned_active);
 4909	INIT_LIST_HEAD(&epc->flexible_active);
 4910	atomic_set(&epc->refcount, 1);
 4911}
 4912
 4913static struct perf_event_context *
 4914alloc_perf_context(struct task_struct *task)
 4915{
 4916	struct perf_event_context *ctx;
 4917
 4918	ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
 4919	if (!ctx)
 4920		return NULL;
 4921
 4922	__perf_event_init_context(ctx);
 4923	if (task)
 4924		ctx->task = get_task_struct(task);
 4925
 4926	return ctx;
 4927}
 4928
 4929static struct task_struct *
 4930find_lively_task_by_vpid(pid_t vpid)
 4931{
 4932	struct task_struct *task;
 4933
 4934	rcu_read_lock();
 4935	if (!vpid)
 4936		task = current;
 4937	else
 4938		task = find_task_by_vpid(vpid);
 4939	if (task)
 4940		get_task_struct(task);
 4941	rcu_read_unlock();
 4942
 4943	if (!task)
 4944		return ERR_PTR(-ESRCH);
 4945
 4946	return task;
 4947}
 4948
 4949/*
 4950 * Returns a matching context with refcount and pincount.
 4951 */
 4952static struct perf_event_context *
 4953find_get_context(struct task_struct *task, struct perf_event *event)
 4954{
 4955	struct perf_event_context *ctx, *clone_ctx = NULL;
 4956	struct perf_cpu_context *cpuctx;
 4957	unsigned long flags;
 4958	int err;
 4959
 4960	if (!task) {
 4961		/* Must be root to operate on a CPU event: */
 4962		err = perf_allow_cpu();
 4963		if (err)
 4964			return ERR_PTR(err);
 4965
 4966		cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);
 4967		ctx = &cpuctx->ctx;
 4968		get_ctx(ctx);
 4969		raw_spin_lock_irqsave(&ctx->lock, flags);
 4970		++ctx->pin_count;
 4971		raw_spin_unlock_irqrestore(&ctx->lock, flags);
 4972
 4973		return ctx;
 4974	}
 4975
 4976	err = -EINVAL;
 4977retry:
 4978	ctx = perf_lock_task_context(task, &flags);
 4979	if (ctx) {
 4980		clone_ctx = unclone_ctx(ctx);
 4981		++ctx->pin_count;
 4982
 4983		raw_spin_unlock_irqrestore(&ctx->lock, flags);
 4984
 4985		if (clone_ctx)
 4986			put_ctx(clone_ctx);
 4987	} else {
 4988		ctx = alloc_perf_context(task);
 4989		err = -ENOMEM;
 4990		if (!ctx)
 4991			goto errout;
 4992
 4993		err = 0;
 4994		mutex_lock(&task->perf_event_mutex);
 4995		/*
 4996		 * If it has already passed perf_event_exit_task().
 4997		 * we must see PF_EXITING, it takes this mutex too.
 4998		 */
 4999		if (task->flags & PF_EXITING)
 5000			err = -ESRCH;
 5001		else if (task->perf_event_ctxp)
 5002			err = -EAGAIN;
 5003		else {
 5004			get_ctx(ctx);
 5005			++ctx->pin_count;
 5006			rcu_assign_pointer(task->perf_event_ctxp, ctx);
 5007		}
 5008		mutex_unlock(&task->perf_event_mutex);
 5009
 5010		if (unlikely(err)) {
 5011			put_ctx(ctx);
 5012
 5013			if (err == -EAGAIN)
 5014				goto retry;
 5015			goto errout;
 5016		}
 5017	}
 5018
 5019	return ctx;
 5020
 5021errout:
 5022	return ERR_PTR(err);
 5023}
 5024
 5025static struct perf_event_pmu_context *
 5026find_get_pmu_context(struct pmu *pmu, struct perf_event_context *ctx,
 5027		     struct perf_event *event)
 5028{
 5029	struct perf_event_pmu_context *new = NULL, *pos = NULL, *epc;
 5030
 5031	if (!ctx->task) {
 5032		/*
 5033		 * perf_pmu_migrate_context() / __perf_pmu_install_event()
 5034		 * relies on the fact that find_get_pmu_context() cannot fail
 5035		 * for CPU contexts.
 5036		 */
 5037		struct perf_cpu_pmu_context *cpc;
 5038
 5039		cpc = *per_cpu_ptr(pmu->cpu_pmu_context, event->cpu);
 5040		epc = &cpc->epc;
 5041		raw_spin_lock_irq(&ctx->lock);
 5042		if (!epc->ctx) {
 5043			/*
 5044			 * One extra reference for the pmu; see perf_pmu_free().
 5045			 */
 5046			atomic_set(&epc->refcount, 2);
 5047			epc->embedded = 1;
 5048			list_add(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list);
 5049			epc->ctx = ctx;
 5050		} else {
 5051			WARN_ON_ONCE(epc->ctx != ctx);
 5052			atomic_inc(&epc->refcount);
 5053		}
 5054		raw_spin_unlock_irq(&ctx->lock);
 5055		return epc;
 5056	}
 5057
 5058	new = kzalloc(sizeof(*epc), GFP_KERNEL);
 5059	if (!new)
 5060		return ERR_PTR(-ENOMEM);
 5061
 5062	__perf_init_event_pmu_context(new, pmu);
 5063
 5064	/*
 5065	 * XXX
 5066	 *
 5067	 * lockdep_assert_held(&ctx->mutex);
 5068	 *
 5069	 * can't because perf_event_init_task() doesn't actually hold the
 5070	 * child_ctx->mutex.
 5071	 */
 5072
 5073	raw_spin_lock_irq(&ctx->lock);
 5074	list_for_each_entry(epc, &ctx->pmu_ctx_list, pmu_ctx_entry) {
 5075		if (epc->pmu == pmu) {
 5076			WARN_ON_ONCE(epc->ctx != ctx);
 5077			atomic_inc(&epc->refcount);
 5078			goto found_epc;
 5079		}
 5080		/* Make sure the pmu_ctx_list is sorted by PMU type: */
 5081		if (!pos && epc->pmu->type > pmu->type)
 5082			pos = epc;
 5083	}
 5084
 5085	epc = new;
 5086	new = NULL;
 5087
 5088	if (!pos)
 5089		list_add_tail(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list);
 5090	else
 5091		list_add(&epc->pmu_ctx_entry, pos->pmu_ctx_entry.prev);
 5092
 5093	epc->ctx = ctx;
 5094
 5095found_epc:
 5096	raw_spin_unlock_irq(&ctx->lock);
 5097	kfree(new);
 5098
 5099	return epc;
 5100}
 5101
 5102static void get_pmu_ctx(struct perf_event_pmu_context *epc)
 5103{
 5104	WARN_ON_ONCE(!atomic_inc_not_zero(&epc->refcount));
 5105}
 5106
 5107static void free_cpc_rcu(struct rcu_head *head)
 5108{
 5109	struct perf_cpu_pmu_context *cpc =
 5110		container_of(head, typeof(*cpc), epc.rcu_head);
 5111
 5112	kfree(cpc);
 5113}
 5114
 5115static void free_epc_rcu(struct rcu_head *head)
 5116{
 5117	struct perf_event_pmu_context *epc = container_of(head, typeof(*epc), rcu_head);
 5118
 5119	kfree(epc);
 5120}
 5121
 5122static void put_pmu_ctx(struct perf_event_pmu_context *epc)
 5123{
 5124	struct perf_event_context *ctx = epc->ctx;
 5125	unsigned long flags;
 5126
 5127	/*
 5128	 * XXX
 5129	 *
 5130	 * lockdep_assert_held(&ctx->mutex);
 5131	 *
 5132	 * can't because of the call-site in _free_event()/put_event()
 5133	 * which isn't always called under ctx->mutex.
 5134	 */
 5135	if (!atomic_dec_and_raw_lock_irqsave(&epc->refcount, &ctx->lock, flags))
 5136		return;
 5137
 5138	WARN_ON_ONCE(list_empty(&epc->pmu_ctx_entry));
 5139
 5140	list_del_init(&epc->pmu_ctx_entry);
 5141	epc->ctx = NULL;
 5142
 5143	WARN_ON_ONCE(!list_empty(&epc->pinned_active));
 5144	WARN_ON_ONCE(!list_empty(&epc->flexible_active));
 5145
 5146	raw_spin_unlock_irqrestore(&ctx->lock, flags);
 5147
 5148	if (epc->embedded) {
 5149		call_rcu(&epc->rcu_head, free_cpc_rcu);
 5150		return;
 5151	}
 5152
 5153	call_rcu(&epc->rcu_head, free_epc_rcu);
 5154}
 5155
 5156static void perf_event_free_filter(struct perf_event *event);
 5157
 5158static void free_event_rcu(struct rcu_head *head)
 5159{
 5160	struct perf_event *event = container_of(head, typeof(*event), rcu_head);
 5161
 5162	if (event->ns)
 5163		put_pid_ns(event->ns);
 5164	perf_event_free_filter(event);
 5165	kmem_cache_free(perf_event_cache, event);
 5166}
 5167
 5168static void ring_buffer_attach(struct perf_event *event,
 5169			       struct perf_buffer *rb);
 5170
 5171static void detach_sb_event(struct perf_event *event)
 5172{
 5173	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);
 5174
 5175	raw_spin_lock(&pel->lock);
 5176	list_del_rcu(&event->sb_list);
 5177	raw_spin_unlock(&pel->lock);
 5178}
 5179
 5180static bool is_sb_event(struct perf_event *event)
 5181{
 5182	struct perf_event_attr *attr = &event->attr;
 5183
 5184	if (event->parent)
 5185		return false;
 5186
 5187	if (event->attach_state & PERF_ATTACH_TASK)
 5188		return false;
 5189
 5190	if (attr->mmap || attr->mmap_data || attr->mmap2 ||
 5191	    attr->comm || attr->comm_exec ||
 5192	    attr->task || attr->ksymbol ||
 5193	    attr->context_switch || attr->text_poke ||
 5194	    attr->bpf_event)
 5195		return true;
 5196
 5197	return false;
 5198}
 5199
 5200static void unaccount_pmu_sb_event(struct perf_event *event)
 5201{
 5202	if (is_sb_event(event))
 5203		detach_sb_event(event);
 5204}
 5205
 5206#ifdef CONFIG_NO_HZ_FULL
 5207static DEFINE_SPINLOCK(nr_freq_lock);
 5208#endif
 5209
 5210static void unaccount_freq_event_nohz(void)
 5211{
 5212#ifdef CONFIG_NO_HZ_FULL
 5213	spin_lock(&nr_freq_lock);
 5214	if (atomic_dec_and_test(&nr_freq_events))
 5215		tick_nohz_dep_clear(TICK_DEP_BIT_PERF_EVENTS);
 5216	spin_unlock(&nr_freq_lock);
 5217#endif
 5218}
 5219
 5220static void unaccount_freq_event(void)
 5221{
 5222	if (tick_nohz_full_enabled())
 5223		unaccount_freq_event_nohz();
 5224	else
 5225		atomic_dec(&nr_freq_events);
 5226}
 5227
 5228
 5229static struct perf_ctx_data *
 5230alloc_perf_ctx_data(struct kmem_cache *ctx_cache, bool global)
 5231{
 5232	struct perf_ctx_data *cd;
 5233
 5234	cd = kzalloc(sizeof(*cd), GFP_KERNEL);
 5235	if (!cd)
 5236		return NULL;
 5237
 5238	cd->data = kmem_cache_zalloc(ctx_cache, GFP_KERNEL);
 5239	if (!cd->data) {
 5240		kfree(cd);
 5241		return NULL;
 5242	}
 5243
 5244	cd->global = global;
 5245	cd->ctx_cache = ctx_cache;
 5246	refcount_set(&cd->refcount, 1);
 5247
 5248	return cd;
 5249}
 5250
 5251static void free_perf_ctx_data(struct perf_ctx_data *cd)
 5252{
 5253	kmem_cache_free(cd->ctx_cache, cd->data);
 5254	kfree(cd);
 5255}
 5256
 5257static void __free_perf_ctx_data_rcu(struct rcu_head *rcu_head)
 5258{
 5259	struct perf_ctx_data *cd;
 5260
 5261	cd = container_of(rcu_head, struct perf_ctx_data, rcu_head);
 5262	free_perf_ctx_data(cd);
 5263}
 5264
 5265static inline void perf_free_ctx_data_rcu(struct perf_ctx_data *cd)
 5266{
 5267	call_rcu(&cd->rcu_head, __free_perf_ctx_data_rcu);
 5268}
 5269
 5270static int
 5271attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache,
 5272		     bool global)
 5273{
 5274	struct perf_ctx_data *cd, *old = NULL;
 5275
 5276	cd = alloc_perf_ctx_data(ctx_cache, global);
 5277	if (!cd)
 5278		return -ENOMEM;
 5279
 5280	for (;;) {
 5281		if (try_cmpxchg(&task->perf_ctx_data, &old, cd)) {
 5282			if (old)
 5283				perf_free_ctx_data_rcu(old);
 5284			/*
 5285			 * Above try_cmpxchg() pairs with try_cmpxchg() from
 5286			 * detach_task_ctx_data() such that
 5287			 * if we race with perf_event_exit_task(), we must
 5288			 * observe PF_EXITING.
 5289			 */
 5290			if (task->flags & PF_EXITING) {
 5291				/* detach_task_ctx_data() may free it already */
 5292				if (try_cmpxchg(&task->perf_ctx_data, &cd, NULL))
 5293					perf_free_ctx_data_rcu(cd);
 5294			}
 5295			return 0;
 5296		}
 5297
 5298		if (!old) {
 5299			/*
 5300			 * After seeing a dead @old, we raced with
 5301			 * removal and lost, try again to install @cd.
 5302			 */
 5303			continue;
 5304		}
 5305
 5306		if (refcount_inc_not_zero(&old->refcount)) {
 5307			free_perf_ctx_data(cd); /* unused */
 5308			return 0;
 5309		}
 5310
 5311		/*
 5312		 * @old is a dead object, refcount==0 is stable, try and
 5313		 * replace it with @cd.
 5314		 */
 5315	}
 5316	return 0;
 5317}
 5318
 5319static void __detach_global_ctx_data(void);
 5320DEFINE_STATIC_PERCPU_RWSEM(global_ctx_data_rwsem);
 5321static refcount_t global_ctx_data_ref;
 5322
 5323static int
 5324attach_global_ctx_data(struct kmem_cache *ctx_cache)
 5325{
 5326	struct task_struct *g, *p;
 5327	struct perf_ctx_data *cd;
 5328	int ret;
 5329
 5330	if (refcount_inc_not_zero(&global_ctx_data_ref))
 5331		return 0;
 5332
 5333	guard(percpu_write)(&global_ctx_data_rwsem);
 5334	if (refcount_inc_not_zero(&global_ctx_data_ref))
 5335		return 0;
 5336again:
 5337	/* Allocate everything */
 5338	scoped_guard (rcu) {
 5339		for_each_process_thread(g, p) {
 5340			if (p->flags & PF_EXITING)
 5341				continue;
 5342			cd = rcu_dereference(p->perf_ctx_data);
 5343			if (cd && !cd->global) {
 5344				cd->global = 1;
 5345				if (!refcount_inc_not_zero(&cd->refcount))
 5346					cd = NULL;
 5347			}
 5348			if (!cd) {
 5349				get_task_struct(p);
 5350				goto alloc;
 5351			}
 5352		}
 5353	}
 5354
 5355	refcount_set(&global_ctx_data_ref, 1);
 5356
 5357	return 0;
 5358alloc:
 5359	ret = attach_task_ctx_data(p, ctx_cache, true);
 5360	put_task_struct(p);
 5361	if (ret) {
 5362		__detach_global_ctx_data();
 5363		return ret;
 5364	}
 5365	goto again;
 5366}
 5367
 5368static int
 5369attach_perf_ctx_data(struct perf_event *event)
 5370{
 5371	struct task_struct *task = event->hw.target;
 5372	struct kmem_cache *ctx_cache = event->pmu->task_ctx_cache;
 5373	int ret;
 5374
 5375	if (!ctx_cache)
 5376		return -ENOMEM;
 5377
 5378	if (task)
 5379		return attach_task_ctx_data(task, ctx_cache, false);
 5380
 5381	ret = attach_global_ctx_data(ctx_cache);
 5382	if (ret)
 5383		return ret;
 5384
 5385	event->attach_state |= PERF_ATTACH_GLOBAL_DATA;
 5386	return 0;
 5387}
 5388
 5389static void
 5390detach_task_ctx_data(struct task_struct *p)
 5391{
 5392	struct perf_ctx_data *cd;
 5393
 5394	scoped_guard (rcu) {
 5395		cd = rcu_dereference(p->perf_ctx_data);
 5396		if (!cd || !refcount_dec_and_test(&cd->refcount))
 5397			return;
 5398	}
 5399
 5400	/*
 5401	 * The old ctx_data may be lost because of the race.
 5402	 * Nothing is required to do for the case.
 5403	 * See attach_task_ctx_data().
 5404	 */
 5405	if (try_cmpxchg((struct perf_ctx_data **)&p->perf_ctx_data, &cd, NULL))
 5406		perf_free_ctx_data_rcu(cd);
 5407}
 5408
 5409static void __detach_global_ctx_data(void)
 5410{
 5411	struct task_struct *g, *p;
 5412	struct perf_ctx_data *cd;
 5413
 5414again:
 5415	scoped_guard (rcu) {
 5416		for_each_process_thread(g, p) {
 5417			cd = rcu_dereference(p->perf_ctx_data);
 5418			if (!cd || !cd->global)
 5419				continue;
 5420			cd->global = 0;
 5421			get_task_struct(p);
 5422			goto detach;
 5423		}
 5424	}
 5425	return;
 5426detach:
 5427	detach_task_ctx_data(p);
 5428	put_task_struct(p);
 5429	goto again;
 5430}
 5431
 5432static void detach_global_ctx_data(void)
 5433{
 5434	if (refcount_dec_not_one(&global_ctx_data_ref))
 5435		return;
 5436
 5437	guard(percpu_write)(&global_ctx_data_rwsem);
 5438	if (!refcount_dec_and_test(&global_ctx_data_ref))
 5439		return;
 5440
 5441	/* remove everything */
 5442	__detach_global_ctx_data();
 5443}
 5444
 5445static void detach_perf_ctx_data(struct perf_event *event)
 5446{
 5447	struct task_struct *task = event->hw.target;
 5448
 5449	event->attach_state &= ~PERF_ATTACH_TASK_DATA;
 5450
 5451	if (task)
 5452		return detach_task_ctx_data(task);
 5453
 5454	if (event->attach_state & PERF_ATTACH_GLOBAL_DATA) {
 5455		detach_global_ctx_data();
 5456		event->attach_state &= ~PERF_ATTACH_GLOBAL_DATA;
 5457	}
 5458}
 5459
 5460static void unaccount_event(struct perf_event *event)
 5461{
 5462	bool dec = false;
 5463
 5464	if (event->parent)
 5465		return;
 5466
 5467	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))
 5468		dec = true;
 5469	if (event->attr.mmap || event->attr.mmap_data)
 5470		atomic_dec(&nr_mmap_events);
 5471	if (event->attr.build_id)
 5472		atomic_dec(&nr_build_id_events);
 5473	if (event->attr.comm)
 5474		atomic_dec(&nr_comm_events);
 5475	if (event->attr.namespaces)
 5476		atomic_dec(&nr_namespaces_events);
 5477	if (event->attr.cgroup)
 5478		atomic_dec(&nr_cgroup_events);
 5479	if (event->attr.task)
 5480		atomic_dec(&nr_task_events);
 5481	if (event->attr.freq)
 5482		unaccount_freq_event();
 5483	if (event->attr.context_switch) {
 5484		dec = true;
 5485		atomic_dec(&nr_switch_events);
 5486	}
 5487	if (is_cgroup_event(event))
 5488		dec = true;
 5489	if (has_branch_stack(event))
 5490		dec = true;
 5491	if (event->attr.ksymbol)
 5492		atomic_dec(&nr_ksymbol_events);
 5493	if (event->attr.bpf_event)
 5494		atomic_dec(&nr_bpf_events);
 5495	if (event->attr.text_poke)
 5496		atomic_dec(&nr_text_poke_events);
 5497
 5498	if (dec) {
 5499		if (!atomic_add_unless(&perf_sched_count, -1, 1))
 5500			schedule_delayed_work(&perf_sched_work, HZ);
 5501	}
 5502
 5503	unaccount_pmu_sb_event(event);
 5504}
 5505
 5506static void perf_sched_delayed(struct work_struct *work)
 5507{
 5508	mutex_lock(&perf_sched_mutex);
 5509	if (atomic_dec_and_test(&perf_sched_count))
 5510		static_branch_disable(&perf_sched_events);
 5511	mutex_unlock(&perf_sched_mutex);
 5512}
 5513
 5514/*
 5515 * The following implement mutual exclusion of events on "exclusive" pmus
 5516 * (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled
 5517 * at a time, so we disallow creating events that might conflict, namely:
 5518 *
 5519 *  1) cpu-wide events in the presence of per-task events,
 5520 *  2) per-task events in the presence of cpu-wide events,
 5521 *  3) two matching events on the same perf_event_context.
 5522 *
 5523 * The former two cases are handled in the allocation path (perf_event_alloc(),
 5524 * _free_event()), the latter -- before the first perf_install_in_context().
 5525 */
 5526static int exclusive_event_init(struct perf_event *event)
 5527{
 5528	struct pmu *pmu = event->pmu;
 5529
 5530	if (!is_exclusive_pmu(pmu))
 5531		return 0;
 5532
 5533	/*
 5534	 * Prevent co-existence of per-task and cpu-wide events on the
 5535	 * same exclusive pmu.
 5536	 *
 5537	 * Negative pmu::exclusive_cnt means there are cpu-wide
 5538	 * events on this "exclusive" pmu, positive means there are
 5539	 * per-task events.
 5540	 *
 5541	 * Since this is called in perf_event_alloc() path, event::ctx
 5542	 * doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK
 5543	 * to mean "per-task event", because unlike other attach states it
 5544	 * never gets cleared.
 5545	 */
 5546	if (event->attach_state & PERF_ATTACH_TASK) {
 5547		if (!atomic_inc_unless_negative(&pmu->exclusive_cnt))
 5548			return -EBUSY;
 5549	} else {
 5550		if (!atomic_dec_unless_positive(&pmu->exclusive_cnt))
 5551			return -EBUSY;
 5552	}
 5553
 5554	event->attach_state |= PERF_ATTACH_EXCLUSIVE;
 5555
 5556	return 0;
 5557}
 5558
 5559static void exclusive_event_destroy(struct perf_event *event)
 5560{
 5561	struct pmu *pmu = event->pmu;
 5562
 5563	/* see comment in exclusive_event_init() */
 5564	if (event->attach_state & PERF_ATTACH_TASK)
 5565		atomic_dec(&pmu->exclusive_cnt);
 5566	else
 5567		atomic_inc(&pmu->exclusive_cnt);
 5568
 5569	event->attach_state &= ~PERF_ATTACH_EXCLUSIVE;
 5570}
 5571
 5572static bool exclusive_event_match(struct perf_event *e1, struct perf_event *e2)
 5573{
 5574	if ((e1->pmu == e2->pmu) &&
 5575	    (e1->cpu == e2->cpu ||
 5576	     e1->cpu == -1 ||
 5577	     e2->cpu == -1))
 5578		return true;
 5579	return false;
 5580}
 5581
 5582static bool exclusive_event_installable(struct perf_event *event,
 5583					struct perf_event_context *ctx)
 5584{
 5585	struct perf_event *iter_event;
 5586	struct pmu *pmu = event->pmu;
 5587
 5588	lockdep_assert_held(&ctx->mutex);
 5589
 5590	if (!is_exclusive_pmu(pmu))
 5591		return true;
 5592
 5593	list_for_each_entry(iter_event, &ctx->event_list, event_entry) {
 5594		if (exclusive_event_match(iter_event, event))
 5595			return false;
 5596	}
 5597
 5598	return true;
 5599}
 5600
 5601static void perf_free_addr_filters(struct perf_event *event);
 5602
 5603/* vs perf_event_alloc() error */
 5604static void __free_event(struct perf_event *event)
 5605{
 5606	struct pmu *pmu = event->pmu;
 5607
 5608	if (event->attach_state & PERF_ATTACH_CALLCHAIN)
 5609		put_callchain_buffers();
 5610
 5611	kfree(event->addr_filter_ranges);
 5612
 5613	if (event->attach_state & PERF_ATTACH_EXCLUSIVE)
 5614		exclusive_event_destroy(event);
 5615
 5616	if (is_cgroup_event(event))
 5617		perf_detach_cgroup(event);
 5618
 5619	if (event->attach_state & PERF_ATTACH_TASK_DATA)
 5620		detach_perf_ctx_data(event);
 5621
 5622	if (event->destroy)
 5623		event->destroy(event);
 5624
 5625	/*
 5626	 * Must be after ->destroy(), due to uprobe_perf_close() using
 5627	 * hw.target.
 5628	 */
 5629	if (event->hw.target)
 5630		put_task_struct(event->hw.target);
 5631
 5632	if (event->pmu_ctx) {
 5633		/*
 5634		 * put_pmu_ctx() needs an event->ctx reference, because of
 5635		 * epc->ctx.
 5636		 */
 5637		WARN_ON_ONCE(!pmu);
 5638		WARN_ON_ONCE(!event->ctx);
 5639		WARN_ON_ONCE(event->pmu_ctx->ctx != event->ctx);
 5640		put_pmu_ctx(event->pmu_ctx);
 5641	}
 5642
 5643	/*
 5644	 * perf_event_free_task() relies on put_ctx() being 'last', in
 5645	 * particular all task references must be cleaned up.
 5646	 */
 5647	if (event->ctx)
 5648		put_ctx(event->ctx);
 5649
 5650	if (pmu) {
 5651		module_put(pmu->module);
 5652		scoped_guard (spinlock, &pmu->events_lock) {
 5653			list_del(&event->pmu_list);
 5654			wake_up_var(pmu);
 5655		}
 5656	}
 5657
 5658	call_rcu(&event->rcu_head, free_event_rcu);
 5659}
 5660
 5661DEFINE_FREE(__free_event, struct perf_event *, if (_T) __free_event(_T))
 5662
 5663/* vs perf_event_alloc() success */
 5664static void _free_event(struct perf_event *event)
 5665{
 5666	irq_work_sync(&event->pending_irq);
 5667	irq_work_sync(&event->pending_disable_irq);
 5668
 5669	unaccount_event(event);
 5670
 5671	security_perf_event_free(event);
 5672
 5673	if (event->rb) {
 5674		/*
 5675		 * Can happen when we close an event with re-directed output.
 5676		 *
 5677		 * Since we have a 0 refcount, perf_mmap_close() will skip
 5678		 * over us; possibly making our ring_buffer_put() the last.
 5679		 */
 5680		mutex_lock(&event->mmap_mutex);
 5681		ring_buffer_attach(event, NULL);
 5682		mutex_unlock(&event->mmap_mutex);
 5683	}
 5684
 5685	perf_event_free_bpf_prog(event);
 5686	perf_free_addr_filters(event);
 5687
 5688	__free_event(event);
 5689}
 5690
 5691/*
 5692 * Used to free events which have a known refcount of 1, such as in error paths
 5693 * of inherited events.
 5694 */
 5695static void free_event(struct perf_event *event)
 5696{
 5697	if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,
 5698				     "unexpected event refcount: %ld; ptr=%p\n",
 5699				     atomic_long_read(&event->refcount), event)) {
 5700		/* leak to avoid use-after-free */
 5701		return;
 5702	}
 5703
 5704	_free_event(event);
 5705}
 5706
 5707/*
 5708 * Remove user event from the owner task.
 5709 */
 5710static void perf_remove_from_owner(struct perf_event *event)
 5711{
 5712	struct task_struct *owner;
 5713
 5714	rcu_read_lock();
 5715	/*
 5716	 * Matches the smp_store_release() in perf_event_exit_task(). If we
 5717	 * observe !owner it means the list deletion is complete and we can
 5718	 * indeed free this event, otherwise we need to serialize on
 5719	 * owner->perf_event_mutex.
 5720	 */
 5721	owner = READ_ONCE(event->owner);
 5722	if (owner) {
 5723		/*
 5724		 * Since delayed_put_task_struct() also drops the last
 5725		 * task reference we can safely take a new reference
 5726		 * while holding the rcu_read_lock().
 5727		 */
 5728		get_task_struct(owner);
 5729	}
 5730	rcu_read_unlock();
 5731
 5732	if (owner) {
 5733		/*
 5734		 * If we're here through perf_event_exit_task() we're already
 5735		 * holding ctx->mutex which would be an inversion wrt. the
 5736		 * normal lock order.
 5737		 *
 5738		 * However we can safely take this lock because its the child
 5739		 * ctx->mutex.
 5740		 */
 5741		mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING);
 5742
 5743		/*
 5744		 * We have to re-check the event->owner field, if it is cleared
 5745		 * we raced with perf_event_exit_task(), acquiring the mutex
 5746		 * ensured they're done, and we can proceed with freeing the
 5747		 * event.
 5748		 */
 5749		if (event->owner) {
 5750			list_del_init(&event->owner_entry);
 5751			smp_store_release(&event->owner, NULL);
 5752		}
 5753		mutex_unlock(&owner->perf_event_mutex);
 5754		put_task_struct(owner);
 5755	}
 5756}
 5757
 5758static void put_event(struct perf_event *event)
 5759{
 5760	struct perf_event *parent;
 5761
 5762	if (!atomic_long_dec_and_test(&event->refcount))
 5763		return;
 5764
 5765	parent = event->parent;
 5766	_free_event(event);
 5767
 5768	/* Matches the refcount bump in inherit_event() */
 5769	if (parent)
 5770		put_event(parent);
 5771}
 5772
 5773/*
 5774 * Kill an event dead; while event:refcount will preserve the event
 5775 * object, it will not preserve its functionality. Once the last 'user'
 5776 * gives up the object, we'll destroy the thing.
 5777 */
 5778int perf_event_release_kernel(struct perf_event *event)
 5779{
 5780	struct perf_event_context *ctx = event->ctx;
 5781	struct perf_event *child, *tmp;
 5782
 5783	/*
 5784	 * If we got here through err_alloc: free_event(event); we will not
 5785	 * have attached to a context yet.
 5786	 */
 5787	if (!ctx) {
 5788		WARN_ON_ONCE(event->attach_state &
 5789				(PERF_ATTACH_CONTEXT|PERF_ATTACH_GROUP));
 5790		goto no_ctx;
 5791	}
 5792
 5793	if (!is_kernel_event(event))
 5794		perf_remove_from_owner(event);
 5795
 5796	ctx = perf_event_ctx_lock(event);
 5797	WARN_ON_ONCE(ctx->parent_ctx);
 5798
 5799	/*
 5800	 * Mark this event as STATE_DEAD, there is no external reference to it
 5801	 * anymore.
 5802	 *
 5803	 * Anybody acquiring event->child_mutex after the below loop _must_
 5804	 * also see this, most importantly inherit_event() which will avoid
 5805	 * placing more children on the list.
 5806	 *
 5807	 * Thus this guarantees that we will in fact observe and kill _ALL_
 5808	 * child events.
 5809	 */
 5810	if (event->state > PERF_EVENT_STATE_REVOKED) {
 5811		perf_remove_from_context(event, DETACH_GROUP|DETACH_DEAD);
 5812	} else {
 5813		event->state = PERF_EVENT_STATE_DEAD;
 5814	}
 5815
 5816	perf_event_ctx_unlock(event, ctx);
 5817
 5818again:
 5819	mutex_lock(&event->child_mutex);
 5820	list_for_each_entry(child, &event->child_list, child_list) {
 5821		/*
 5822		 * Cannot change, child events are not migrated, see the
 5823		 * comment with perf_event_ctx_lock_nested().
 5824		 */
 5825		ctx = READ_ONCE(child->ctx);
 5826		/*
 5827		 * Since child_mutex nests inside ctx::mutex, we must jump
 5828		 * through hoops. We start by grabbing a reference on the ctx.
 5829		 *
 5830		 * Since the event cannot get freed while we hold the
 5831		 * child_mutex, the context must also exist and have a !0
 5832		 * reference count.
 5833		 */
 5834		get_ctx(ctx);
 5835
 5836		/*
 5837		 * Now that we have a ctx ref, we can drop child_mutex, and
 5838		 * acquire ctx::mutex without fear of it going away. Then we
 5839		 * can re-acquire child_mutex.
 5840		 */
 5841		mutex_unlock(&event->child_mutex);
 5842		mutex_lock(&ctx->mutex);
 5843		mutex_lock(&event->child_mutex);
 5844
 5845		/*
 5846		 * Now that we hold ctx::mutex and child_mutex, revalidate our
 5847		 * state, if child is still the first entry, it didn't get freed
 5848		 * and we can continue doing so.
 5849		 */
 5850		tmp = list_first_entry_or_null(&event->child_list,
 5851					       struct perf_event, child_list);
 5852		if (tmp == child) {
 5853			perf_remove_from_context(child, DETACH_GROUP | DETACH_CHILD);
 5854		} else {
 5855			child = NULL;
 5856		}
 5857
 5858		mutex_unlock(&event->child_mutex);
 5859		mutex_unlock(&ctx->mutex);
 5860
 5861		if (child) {
 5862			/* Last reference unless ->pending_task work is pending */
 5863			put_event(child);
 5864		}
 5865		put_ctx(ctx);
 5866
 5867		goto again;
 5868	}
 5869	mutex_unlock(&event->child_mutex);
 5870
 5871no_ctx:
 5872	/*
 5873	 * Last reference unless ->pending_task work is pending on this event
 5874	 * or any of its children.
 5875	 */
 5876	put_event(event);
 5877	return 0;
 5878}
 5879EXPORT_SYMBOL_GPL(perf_event_release_kernel);
 5880
 5881/*
 5882 * Called when the last reference to the file is gone.
 5883 */
 5884static int perf_release(struct inode *inode, struct file *file)
 5885{
 5886	perf_event_release_kernel(file->private_data);
 5887	return 0;
 5888}
 5889
 5890static u64 __perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
 5891{
 5892	struct perf_event *child;
 5893	u64 total = 0;
 5894
 5895	*enabled = 0;
 5896	*running = 0;
 5897
 5898	mutex_lock(&event->child_mutex);
 5899
 5900	(void)perf_event_read(event, false);
 5901	total += perf_event_count(event, false);
 5902
 5903	*enabled += event->total_time_enabled +
 5904			atomic64_read(&event->child_total_time_enabled);
 5905	*running += event->total_time_running +
 5906			atomic64_read(&event->child_total_time_running);
 5907
 5908	list_for_each_entry(child, &event->child_list, child_list) {
 5909		(void)perf_event_read(child, false);
 5910		total += perf_event_count(child, false);
 5911		*enabled += child->total_time_enabled;
 5912		*running += child->total_time_running;
 5913	}
 5914	mutex_unlock(&event->child_mutex);
 5915
 5916	return total;
 5917}
 5918
 5919u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
 5920{
 5921	struct perf_event_context *ctx;
 5922	u64 count;
 5923
 5924	ctx = perf_event_ctx_lock(event);
 5925	count = __perf_event_read_value(event, enabled, running);
 5926	perf_event_ctx_unlock(event, ctx);
 5927
 5928	return count;
 5929}
 5930EXPORT_SYMBOL_GPL(perf_event_read_value);
 5931
 5932static int __perf_read_group_add(struct perf_event *leader,
 5933					u64 read_format, u64 *values)
 5934{
 5935	struct perf_event_context *ctx = leader->ctx;
 5936	struct perf_event *sub, *parent;
 5937	unsigned long flags;
 5938	int n = 1; /* skip @nr */
 5939	int ret;
 5940
 5941	ret = perf_event_read(leader, true);
 5942	if (ret)
 5943		return ret;
 5944
 5945	raw_spin_lock_irqsave(&ctx->lock, flags);
 5946	/*
 5947	 * Verify the grouping between the parent and child (inherited)
 5948	 * events is still in tact.
 5949	 *
 5950	 * Specifically:
 5951	 *  - leader->ctx->lock pins leader->sibling_list
 5952	 *  - parent->child_mutex pins parent->child_list
 5953	 *  - parent->ctx->mutex pins parent->sibling_list
 5954	 *
 5955	 * Because parent->ctx != leader->ctx (and child_list nests inside
 5956	 * ctx->mutex), group destruction is not atomic between children, also
 5957	 * see perf_event_release_kernel(). Additionally, parent can grow the
 5958	 * group.
 5959	 *
 5960	 * Therefore it is possible to have parent and child groups in a
 5961	 * different configuration and summing over such a beast makes no sense
 5962	 * what so ever.
 5963	 *
 5964	 * Reject this.
 5965	 */
 5966	parent = leader->parent;
 5967	if (parent &&
 5968	    (parent->group_generation != leader->group_generation ||
 5969	     parent->nr_siblings != leader->nr_siblings)) {
 5970		ret = -ECHILD;
 5971		goto unlock;
 5972	}
 5973
 5974	/*
 5975	 * Since we co-schedule groups, {enabled,running} times of siblings
 5976	 * will be identical to those of the leader, so we only publish one
 5977	 * set.
 5978	 */
 5979	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
 5980		values[n++] += leader->total_time_enabled +
 5981			atomic64_read(&leader->child_total_time_enabled);
 5982	}
 5983
 5984	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
 5985		values[n++] += leader->total_time_running +
 5986			atomic64_read(&leader->child_total_time_running);
 5987	}
 5988
 5989	/*
 5990	 * Write {count,id} tuples for every sibling.
 5991	 */
 5992	values[n++] += perf_event_count(leader, false);
 5993	if (read_format & PERF_FORMAT_ID)
 5994		values[n++] = primary_event_id(leader);
 5995	if (read_format & PERF_FORMAT_LOST)
 5996		values[n++] = atomic64_read(&leader->lost_samples);
 5997
 5998	for_each_sibling_event(sub, leader) {
 5999		values[n++] += perf_event_count(sub, false);
 6000		if (read_format & PERF_FORMAT_ID)
 6001			values[n++] = primary_event_id(sub);
 6002		if (read_format & PERF_FORMAT_LOST)
 6003			values[n++] = atomic64_read(&sub->lost_samples);
 6004	}
 6005
 6006unlock:
 6007	raw_spin_unlock_irqrestore(&ctx->lock, flags);
 6008	return ret;
 6009}
 6010
 6011static int perf_read_group(struct perf_event *event,
 6012				   u64 read_format, char __user *buf)
 6013{
 6014	struct perf_event *leader = event->group_leader, *child;
 6015	struct perf_event_context *ctx = leader->ctx;
 6016	int ret;
 6017	u64 *values;
 6018
 6019	lockdep_assert_held(&ctx->mutex);
 6020
 6021	values = kzalloc(event->read_size, GFP_KERNEL);
 6022	if (!values)
 6023		return -ENOMEM;
 6024
 6025	values[0] = 1 + leader->nr_siblings;
 6026
 6027	mutex_lock(&leader->child_mutex);
 6028
 6029	ret = __perf_read_group_add(leader, read_format, values);
 6030	if (ret)
 6031		goto unlock;
 6032
 6033	list_for_each_entry(child, &leader->child_list, child_list) {
 6034		ret = __perf_read_group_add(child, read_format, values);
 6035		if (ret)
 6036			goto unlock;
 6037	}
 6038
 6039	mutex_unlock(&leader->child_mutex);
 6040
 6041	ret = event->read_size;
 6042	if (copy_to_user(buf, values, event->read_size))
 6043		ret = -EFAULT;
 6044	goto out;
 6045
 6046unlock:
 6047	mutex_unlock(&leader->child_mutex);
 6048out:
 6049	kfree(values);
 6050	return ret;
 6051}
 6052
 6053static int perf_read_one(struct perf_event *event,
 6054				 u64 read_format, char __user *buf)
 6055{
 6056	u64 enabled, running;
 6057	u64 values[5];
 6058	int n = 0;
 6059
 6060	values[n++] = __perf_event_read_value(event, &enabled, &running);
 6061	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
 6062		values[n++] = enabled;
 6063	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
 6064		values[n++] = running;
 6065	if (read_format & PERF_FORMAT_ID)
 6066		values[n++] = primary_event_id(event);
 6067	if (read_format & PERF_FORMAT_LOST)
 6068		values[n++] = atomic64_read(&event->lost_samples);
 6069
 6070	if (copy_to_user(buf, values, n * sizeof(u64)))
 6071		return -EFAULT;
 6072
 6073	return n * sizeof(u64);
 6074}
 6075
 6076static bool is_event_hup(struct perf_event *event)
 6077{
 6078	bool no_children;
 6079
 6080	if (event->state > PERF_EVENT_STATE_EXIT)
 6081		return false;
 6082
 6083	mutex_lock(&event->child_mutex);
 6084	no_children = list_empty(&event->child_list);
 6085	mutex_unlock(&event->child_mutex);
 6086	return no_children;
 6087}
 6088
 6089/*
 6090 * Read the performance event - simple non blocking version for now
 6091 */
 6092static ssize_t
 6093__perf_read(struct perf_event *event, char __user *buf, size_t count)
 6094{
 6095	u64 read_format = event->attr.read_format;
 6096	int ret;
 6097
 6098	/*
 6099	 * Return end-of-file for a read on an event that is in
 6100	 * error state (i.e. because it was pinned but it couldn't be
 6101	 * scheduled on to the CPU at some point).
 6102	 */
 6103	if (event->state == PERF_EVENT_STATE_ERROR)
 6104		return 0;
 6105
 6106	if (count < event->read_size)
 6107		return -ENOSPC;
 6108
 6109	WARN_ON_ONCE(event->ctx->parent_ctx);
 6110	if (read_format & PERF_FORMAT_GROUP)
 6111		ret = perf_read_group(event, read_format, buf);
 6112	else
 6113		ret = perf_read_one(event, read_format, buf);
 6114
 6115	return ret;
 6116}
 6117
 6118static ssize_t
 6119perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
 6120{
 6121	struct perf_event *event = file->private_data;
 6122	struct perf_event_context *ctx;
 6123	int ret;
 6124
 6125	ret = security_perf_event_read(event);
 6126	if (ret)
 6127		return ret;
 6128
 6129	ctx = perf_event_ctx_lock(event);
 6130	ret = __perf_read(event, buf, count);
 6131	perf_event_ctx_unlock(event, ctx);
 6132
 6133	return ret;
 6134}
 6135
 6136static __poll_t perf_poll(struct file *file, poll_table *wait)
 6137{
 6138	struct perf_event *event = file->private_data;
 6139	struct perf_buffer *rb;
 6140	__poll_t events = EPOLLHUP;
 6141
 6142	if (event->state <= PERF_EVENT_STATE_REVOKED)
 6143		return EPOLLERR;
 6144
 6145	poll_wait(file, &event->waitq, wait);
 6146
 6147	if (event->state <= PERF_EVENT_STATE_REVOKED)
 6148		return EPOLLERR;
 6149
 6150	if (is_event_hup(event))
 6151		return events;
 6152
 6153	if (unlikely(READ_ONCE(event->state) == PERF_EVENT_STATE_ERROR &&
 6154		     event->attr.pinned))
 6155		return EPOLLERR;
 6156
 6157	/*
 6158	 * Pin the event->rb by taking event->mmap_mutex; otherwise
 6159	 * perf_event_set_output() can swizzle our rb and make us miss wakeups.
 6160	 */
 6161	mutex_lock(&event->mmap_mutex);
 6162	rb = event->rb;
 6163	if (rb)
 6164		events = atomic_xchg(&rb->poll, 0);
 6165	mutex_unlock(&event->mmap_mutex);
 6166	return events;
 6167}
 6168
 6169static void _perf_event_reset(struct perf_event *event)
 6170{
 6171	(void)perf_event_read(event, false);
 6172	local64_set(&event->count, 0);
 6173	perf_event_update_userpage(event);
 6174}
 6175
 6176/* Assume it's not an event with inherit set. */
 6177u64 perf_event_pause(struct perf_event *event, bool reset)
 6178{
 6179	struct perf_event_context *ctx;
 6180	u64 count;
 6181
 6182	ctx = perf_event_ctx_lock(event);
 6183	WARN_ON_ONCE(event->attr.inherit);
 6184	_perf_event_disable(event);
 6185	count = local64_read(&event->count);
 6186	if (reset)
 6187		local64_set(&event->count, 0);
 6188	perf_event_ctx_unlock(event, ctx);
 6189
 6190	return count;
 6191}
 6192EXPORT_SYMBOL_GPL(perf_event_pause);
 6193
 6194/*
 6195 * Holding the top-level event's child_mutex means that any
 6196 * descendant process that has inherited this event will block
 6197 * in perf_event_exit_event() if it goes to exit, thus satisfying the
 6198 * task existence requirements of perf_event_enable/disable.
 6199 */
 6200static void perf_event_for_each_child(struct perf_event *event,
 6201					void (*func)(struct perf_event *))
 6202{
 6203	struct perf_event *child;
 6204
 6205	WARN_ON_ONCE(event->ctx->parent_ctx);
 6206
 6207	mutex_lock(&event->child_mutex);
 6208	func(event);
 6209	list_for_each_entry(child, &event->child_list, child_list)
 6210		func(child);
 6211	mutex_unlock(&event->child_mutex);
 6212}
 6213
 6214static void perf_event_for_each(struct perf_event *event,
 6215				  void (*func)(struct perf_event *))
 6216{
 6217	struct perf_event_context *ctx = event->ctx;
 6218	struct perf_event *sibling;
 6219
 6220	lockdep_assert_held(&ctx->mutex);
 6221
 6222	event = event->group_leader;
 6223
 6224	perf_event_for_each_child(event, func);
 6225	for_each_sibling_event(sibling, event)
 6226		perf_event_for_each_child(sibling, func);
 6227}
 6228
 6229static void __perf_event_period(struct perf_event *event,
 6230				struct perf_cpu_context *cpuctx,
 6231				struct perf_event_context *ctx,
 6232				void *info)
 6233{
 6234	u64 value = *((u64 *)info);
 6235	bool active;
 6236
 6237	if (event->attr.freq) {
 6238		event->attr.sample_freq = value;
 6239	} else {
 6240		event->attr.sample_period = value;
 6241		event->hw.sample_period = value;
 6242	}
 6243
 6244	active = (event->state == PERF_EVENT_STATE_ACTIVE);
 6245	if (active) {
 6246		perf_pmu_disable(event->pmu);
 6247		event->pmu->stop(event, PERF_EF_UPDATE);
 6248	}
 6249
 6250	local64_set(&event->hw.period_left, 0);
 6251
 6252	if (active) {
 6253		event->pmu->start(event, PERF_EF_RELOAD);
 6254		/*
 6255		 * Once the period is force-reset, the event starts immediately.
 6256		 * But the event/group could be throttled. Unthrottle the
 6257		 * event/group now to avoid the next tick trying to unthrottle
 6258		 * while we already re-started the event/group.
 6259		 */
 6260		if (event->hw.interrupts == MAX_INTERRUPTS)
 6261			perf_event_unthrottle_group(event, true);
 6262		perf_pmu_enable(event->pmu);
 6263	}
 6264}
 6265
 6266static int perf_event_check_period(struct perf_event *event, u64 value)
 6267{
 6268	return event->pmu->check_period(event, value);
 6269}
 6270
 6271static int _perf_event_period(struct perf_event *event, u64 value)
 6272{
 6273	if (!is_sampling_event(event))
 6274		return -EINVAL;
 6275
 6276	if (!value)
 6277		return -EINVAL;
 6278
 6279	if (event->attr.freq) {
 6280		if (value > sysctl_perf_event_sample_rate)
 6281			return -EINVAL;
 6282	} else {
 6283		if (perf_event_check_period(event, value))
 6284			return -EINVAL;
 6285		if (value & (1ULL << 63))
 6286			return -EINVAL;
 6287	}
 6288
 6289	event_function_call(event, __perf_event_period, &value);
 6290
 6291	return 0;
 6292}
 6293
 6294int perf_event_period(struct perf_event *event, u64 value)
 6295{
 6296	struct perf_event_context *ctx;
 6297	int ret;
 6298
 6299	ctx = perf_event_ctx_lock(event);
 6300	ret = _perf_event_period(event, value);
 6301	perf_event_ctx_unlock(event, ctx);
 6302
 6303	return ret;
 6304}
 6305EXPORT_SYMBOL_GPL(perf_event_period);
 6306
 6307static const struct file_operations perf_fops;
 6308
 6309static inline bool is_perf_file(struct fd f)
 6310{
 6311	return !fd_empty(f) && fd_file(f)->f_op == &perf_fops;
 6312}
 6313
 6314static int perf_event_set_output(struct perf_event *event,
 6315				 struct perf_event *output_event);
 6316static int perf_event_set_filter(struct perf_event *event, void __user *arg);
 6317static int perf_copy_attr(struct perf_event_attr __user *uattr,
 6318			  struct perf_event_attr *attr);
 6319static int __perf_event_set_bpf_prog(struct perf_event *event,
 6320				     struct bpf_prog *prog,
 6321				     u64 bpf_cookie);
 6322
 6323static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)
 6324{
 6325	void (*func)(struct perf_event *);
 6326	u32 flags = arg;
 6327
 6328	if (event->state <= PERF_EVENT_STATE_REVOKED)
 6329		return -ENODEV;
 6330
 6331	switch (cmd) {
 6332	case PERF_EVENT_IOC_ENABLE:
 6333		func = _perf_event_enable;
 6334		break;
 6335	case PERF_EVENT_IOC_DISABLE:
 6336		func = _perf_event_disable;
 6337		break;
 6338	case PERF_EVENT_IOC_RESET:
 6339		func = _perf_event_reset;
 6340		break;
 6341
 6342	case PERF_EVENT_IOC_REFRESH:
 6343		return _perf_event_refresh(event, arg);
 6344
 6345	case PERF_EVENT_IOC_PERIOD:
 6346	{
 6347		u64 value;
 6348
 6349		if (copy_from_user(&value, (u64 __user *)arg, sizeof(value)))
 6350			return -EFAULT;
 6351
 6352		return _perf_event_period(event, value);
 6353	}
 6354	case PERF_EVENT_IOC_ID:
 6355	{
 6356		u64 id = primary_event_id(event);
 6357
 6358		if (copy_to_user((void __user *)arg, &id, sizeof(id)))
 6359			return -EFAULT;
 6360		return 0;
 6361	}
 6362
 6363	case PERF_EVENT_IOC_SET_OUTPUT:
 6364	{
 6365		CLASS(fd, output)(arg);	     // arg == -1 => empty
 6366		struct perf_event *output_event = NULL;
 6367		if (arg != -1) {
 6368			if (!is_perf_file(output))
 6369				return -EBADF;
 6370			output_event = fd_file(output)->private_data;
 6371		}
 6372		return perf_event_set_output(event, output_event);
 6373	}
 6374
 6375	case PERF_EVENT_IOC_SET_FILTER:
 6376		return perf_event_set_filter(event, (void __user *)arg);
 6377
 6378	case PERF_EVENT_IOC_SET_BPF:
 6379	{
 6380		struct bpf_prog *prog;
 6381		int err;
 6382
 6383		prog = bpf_prog_get(arg);
 6384		if (IS_ERR(prog))
 6385			return PTR_ERR(prog);
 6386
 6387		err = __perf_event_set_bpf_prog(event, prog, 0);
 6388		if (err) {
 6389			bpf_prog_put(prog);
 6390			return err;
 6391		}
 6392
 6393		return 0;
 6394	}
 6395
 6396	case PERF_EVENT_IOC_PAUSE_OUTPUT: {
 6397		struct perf_buffer *rb;
 6398
 6399		rcu_read_lock();
 6400		rb = rcu_dereference(event->rb);
 6401		if (!rb || !rb->nr_pages) {
 6402			rcu_read_unlock();
 6403			return -EINVAL;
 6404		}
 6405		rb_toggle_paused(rb, !!arg);
 6406		rcu_read_unlock();
 6407		return 0;
 6408	}
 6409
 6410	case PERF_EVENT_IOC_QUERY_BPF:
 6411		return perf_event_query_prog_array(event, (void __user *)arg);
 6412
 6413	case PERF_EVENT_IOC_MODIFY_ATTRIBUTES: {
 6414		struct perf_event_attr new_attr;
 6415		int err = perf_copy_attr((struct perf_event_attr __user *)arg,
 6416					 &new_attr);
 6417
 6418		if (err)
 6419			return err;
 6420
 6421		return perf_event_modify_attr(event,  &new_attr);
 6422	}
 6423	default:
 6424		return -ENOTTY;
 6425	}
 6426
 6427	if (flags & PERF_IOC_FLAG_GROUP)
 6428		perf_event_for_each(event, func);
 6429	else
 6430		perf_event_for_each_child(event, func);
 6431
 6432	return 0;
 6433}
 6434
 6435static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 6436{
 6437	struct perf_event *event = file->private_data;
 6438	struct perf_event_context *ctx;
 6439	long ret;
 6440
 6441	/* Treat ioctl like writes as it is likely a mutating operation. */
 6442	ret = security_perf_event_write(event);
 6443	if (ret)
 6444		return ret;
 6445
 6446	ctx = perf_event_ctx_lock(event);
 6447	ret = _perf_ioctl(event, cmd, arg);
 6448	perf_event_ctx_unlock(event, ctx);
 6449
 6450	return ret;
 6451}
 6452
 6453#ifdef CONFIG_COMPAT
 6454static long perf_compat_ioctl(struct file *file, unsigned int cmd,
 6455				unsigned long arg)
 6456{
 6457	switch (_IOC_NR(cmd)) {
 6458	case _IOC_NR(PERF_EVENT_IOC_SET_FILTER):
 6459	case _IOC_NR(PERF_EVENT_IOC_ID):
 6460	case _IOC_NR(PERF_EVENT_IOC_QUERY_BPF):
 6461	case _IOC_NR(PERF_EVENT_IOC_MODIFY_ATTRIBUTES):
 6462		/* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */
 6463		if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) {
 6464			cmd &= ~IOCSIZE_MASK;
 6465			cmd |= sizeof(void *) << IOCSIZE_SHIFT;
 6466		}
 6467		break;
 6468	}
 6469	return perf_ioctl(file, cmd, arg);
 6470}
 6471#else
 6472# define perf_compat_ioctl NULL
 6473#endif
 6474
 6475int perf_event_task_enable(void)
 6476{
 6477	struct perf_event_context *ctx;
 6478	struct perf_event *event;
 6479
 6480	mutex_lock(&current->perf_event_mutex);
 6481	list_for_each_entry(event, &current->perf_event_list, owner_entry) {
 6482		ctx = perf_event_ctx_lock(event);
 6483		perf_event_for_each_child(event, _perf_event_enable);
 6484		perf_event_ctx_unlock(event, ctx);
 6485	}
 6486	mutex_unlock(&current->perf_event_mutex);
 6487
 6488	return 0;
 6489}
 6490
 6491int perf_event_task_disable(void)
 6492{
 6493	struct perf_event_context *ctx;
 6494	struct perf_event *event;
 6495
 6496	mutex_lock(&current->perf_event_mutex);
 6497	list_for_each_entry(event, &current->perf_event_list, owner_entry) {
 6498		ctx = perf_event_ctx_lock(event);
 6499		perf_event_for_each_child(event, _perf_event_disable);
 6500		perf_event_ctx_unlock(event, ctx);
 6501	}
 6502	mutex_unlock(&current->perf_event_mutex);
 6503
 6504	return 0;
 6505}
 6506
 6507static int perf_event_index(struct perf_event *event)
 6508{
 6509	if (event->hw.state & PERF_HES_STOPPED)
 6510		return 0;
 6511
 6512	if (event->state != PERF_EVENT_STATE_ACTIVE)
 6513		return 0;
 6514
 6515	return event->pmu->event_idx(event);
 6516}
 6517
 6518static void perf_event_init_userpage(struct perf_event *event)
 6519{
 6520	struct perf_event_mmap_page *userpg;
 6521	struct perf_buffer *rb;
 6522
 6523	rcu_read_lock();
 6524	rb = rcu_dereference(event->rb);
 6525	if (!rb)
 6526		goto unlock;
 6527
 6528	userpg = rb->user_page;
 6529
 6530	/* Allow new userspace to detect that bit 0 is deprecated */
 6531	userpg->cap_bit0_is_deprecated = 1;
 6532	userpg->size = offsetof(struct perf_event_mmap_page, __reserved);
 6533	userpg->data_offset = PAGE_SIZE;
 6534	userpg->data_size = perf_data_size(rb);
 6535
 6536unlock:
 6537	rcu_read_unlock();
 6538}
 6539
 6540void __weak arch_perf_update_userpage(
 6541	struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now)
 6542{
 6543}
 6544
 6545/*
 6546 * Callers need to ensure there can be no nesting of this function, otherwise
 6547 * the seqlock logic goes bad. We can not serialize this because the arch
 6548 * code calls this from NMI context.
 6549 */
 6550void perf_event_update_userpage(struct perf_event *event)
 6551{
 6552	struct perf_event_mmap_page *userpg;
 6553	struct perf_buffer *rb;
 6554	u64 enabled, running, now;
 6555
 6556	rcu_read_lock();
 6557	rb = rcu_dereference(event->rb);
 6558	if (!rb)
 6559		goto unlock;
 6560
 6561	/*
 6562	 * compute total_time_enabled, total_time_running
 6563	 * based on snapshot values taken when the event
 6564	 * was last scheduled in.
 6565	 *
 6566	 * we cannot simply called update_context_time()
 6567	 * because of locking issue as we can be called in
 6568	 * NMI context
 6569	 */
 6570	calc_timer_values(event, &now, &enabled, &running);
 6571
 6572	userpg = rb->user_page;
 6573	/*
 6574	 * Disable preemption to guarantee consistent time stamps are stored to
 6575	 * the user page.
 6576	 */
 6577	preempt_disable();
 6578	++userpg->lock;
 6579	barrier();
 6580	userpg->index = perf_event_index(event);
 6581	userpg->offset = perf_event_count(event, false);
 6582	if (userpg->index)
 6583		userpg->offset -= local64_read(&event->hw.prev_count);
 6584
 6585	userpg->time_enabled = enabled +
 6586			atomic64_read(&event->child_total_time_enabled);
 6587
 6588	userpg->time_running = running +
 6589			atomic64_read(&event->child_total_time_running);
 6590
 6591	arch_perf_update_userpage(event, userpg, now);
 6592
 6593	barrier();
 6594	++userpg->lock;
 6595	preempt_enable();
 6596unlock:
 6597	rcu_read_unlock();
 6598}
 6599EXPORT_SYMBOL_GPL(perf_event_update_userpage);
 6600
 6601static void ring_buffer_attach(struct perf_event *event,
 6602			       struct perf_buffer *rb)
 6603{
 6604	struct perf_buffer *old_rb = NULL;
 6605	unsigned long flags;
 6606
 6607	WARN_ON_ONCE(event->parent);
 6608
 6609	if (event->rb) {
 6610		/*
 6611		 * Should be impossible, we set this when removing
 6612		 * event->rb_entry and wait/clear when adding event->rb_entry.
 6613		 */
 6614		WARN_ON_ONCE(event->rcu_pending);
 6615
 6616		old_rb = event->rb;
 6617		spin_lock_irqsave(&old_rb->event_lock, flags);
 6618		list_del_rcu(&event->rb_entry);
 6619		spin_unlock_irqrestore(&old_rb->event_lock, flags);
 6620
 6621		event->rcu_batches = get_state_synchronize_rcu();
 6622		event->rcu_pending = 1;
 6623	}
 6624
 6625	if (rb) {
 6626		if (event->rcu_pending) {
 6627			cond_synchronize_rcu(event->rcu_batches);
 6628			event->rcu_pending = 0;
 6629		}
 6630
 6631		spin_lock_irqsave(&rb->event_lock, flags);
 6632		list_add_rcu(&event->rb_entry, &rb->event_list);
 6633		spin_unlock_irqrestore(&rb->event_lock, flags);
 6634	}
 6635
 6636	/*
 6637	 * Avoid racing with perf_mmap_close(AUX): stop the event
 6638	 * before swizzling the event::rb pointer; if it's getting
 6639	 * unmapped, its aux_mmap_count will be 0 and it won't
 6640	 * restart. See the comment in __perf_pmu_output_stop().
 6641	 *
 6642	 * Data will inevitably be lost when set_output is done in
 6643	 * mid-air, but then again, whoever does it like this is
 6644	 * not in for the data anyway.
 6645	 */
 6646	if (has_aux(event))
 6647		perf_event_stop(event, 0);
 6648
 6649	rcu_assign_pointer(event->rb, rb);
 6650
 6651	if (old_rb) {
 6652		ring_buffer_put(old_rb);
 6653		/*
 6654		 * Since we detached before setting the new rb, so that we
 6655		 * could attach the new rb, we could have missed a wakeup.
 6656		 * Provide it now.
 6657		 */
 6658		wake_up_all(&event->waitq);
 6659	}
 6660}
 6661
 6662static void ring_buffer_wakeup(struct perf_event *event)
 6663{
 6664	struct perf_buffer *rb;
 6665
 6666	if (event->parent)
 6667		event = event->parent;
 6668
 6669	rcu_read_lock();
 6670	rb = rcu_dereference(event->rb);
 6671	if (rb) {
 6672		list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
 6673			wake_up_all(&event->waitq);
 6674	}
 6675	rcu_read_unlock();
 6676}
 6677
 6678struct perf_buffer *ring_buffer_get(struct perf_event *event)
 6679{
 6680	struct perf_buffer *rb;
 6681
 6682	if (event->parent)
 6683		event = event->parent;
 6684
 6685	rcu_read_lock();
 6686	rb = rcu_dereference(event->rb);
 6687	if (rb) {
 6688		if (!refcount_inc_not_zero(&rb->refcount))
 6689			rb = NULL;
 6690	}
 6691	rcu_read_unlock();
 6692
 6693	return rb;
 6694}
 6695
 6696void ring_buffer_put(struct perf_buffer *rb)
 6697{
 6698	if (!refcount_dec_and_test(&rb->refcount))
 6699		return;
 6700
 6701	WARN_ON_ONCE(!list_empty(&rb->event_list));
 6702
 6703	call_rcu(&rb->rcu_head, rb_free_rcu);
 6704}
 6705
 6706typedef void (*mapped_f)(struct perf_event *event, struct mm_struct *mm);
 6707
 6708#define get_mapped(event, func)			\
 6709({	struct pmu *pmu;			\
 6710	mapped_f f = NULL;			\
 6711	guard(rcu)();				\
 6712	pmu = READ_ONCE(event->pmu);		\
 6713	if (pmu)				\
 6714		f = pmu->func;			\
 6715	f;					\
 6716})
 6717
 6718static void perf_mmap_open(struct vm_area_struct *vma)
 6719{
 6720	struct perf_event *event = vma->vm_file->private_data;
 6721	mapped_f mapped = get_mapped(event, event_mapped);
 6722
 6723	refcount_inc(&event->mmap_count);
 6724	refcount_inc(&event->rb->mmap_count);
 6725
 6726	if (vma->vm_pgoff)
 6727		refcount_inc(&event->rb->aux_mmap_count);
 6728
 6729	if (mapped)
 6730		mapped(event, vma->vm_mm);
 6731}
 6732
 6733static void perf_pmu_output_stop(struct perf_event *event);
 6734
 6735/*
 6736 * A buffer can be mmap()ed multiple times; either directly through the same
 6737 * event, or through other events by use of perf_event_set_output().
 6738 *
 6739 * In order to undo the VM accounting done by perf_mmap() we need to destroy
 6740 * the buffer here, where we still have a VM context. This means we need
 6741 * to detach all events redirecting to us.
 6742 */
 6743static void perf_mmap_close(struct vm_area_struct *vma)
 6744{
 6745	struct perf_event *event = vma->vm_file->private_data;
 6746	mapped_f unmapped = get_mapped(event, event_unmapped);
 6747	struct perf_buffer *rb = ring_buffer_get(event);
 6748	struct user_struct *mmap_user = rb->mmap_user;
 6749	int mmap_locked = rb->mmap_locked;
 6750	unsigned long size = perf_data_size(rb);
 6751	bool detach_rest = false;
 6752
 6753	/* FIXIES vs perf_pmu_unregister() */
 6754	if (unmapped)
 6755		unmapped(event, vma->vm_mm);
 6756
 6757	/*
 6758	 * The AUX buffer is strictly a sub-buffer, serialize using aux_mutex
 6759	 * to avoid complications.
 6760	 */
 6761	if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff &&
 6762	    refcount_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) {
 6763		/*
 6764		 * Stop all AUX events that are writing to this buffer,
 6765		 * so that we can free its AUX pages and corresponding PMU
 6766		 * data. Note that after rb::aux_mmap_count dropped to zero,
 6767		 * they won't start any more (see perf_aux_output_begin()).
 6768		 */
 6769		perf_pmu_output_stop(event);
 6770
 6771		/* now it's safe to free the pages */
 6772		atomic_long_sub(rb->aux_nr_pages - rb->aux_mmap_locked, &mmap_user->locked_vm);
 6773		atomic64_sub(rb->aux_mmap_locked, &vma->vm_mm->pinned_vm);
 6774
 6775		/* this has to be the last one */
 6776		rb_free_aux(rb);
 6777		WARN_ON_ONCE(refcount_read(&rb->aux_refcount));
 6778
 6779		mutex_unlock(&rb->aux_mutex);
 6780	}
 6781
 6782	if (refcount_dec_and_test(&rb->mmap_count))
 6783		detach_rest = true;
 6784
 6785	if (!refcount_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
 6786		goto out_put;
 6787
 6788	ring_buffer_attach(event, NULL);
 6789	mutex_unlock(&event->mmap_mutex);
 6790
 6791	/* If there's still other mmap()s of this buffer, we're done. */
 6792	if (!detach_rest)
 6793		goto out_put;
 6794
 6795	/*
 6796	 * No other mmap()s, detach from all other events that might redirect
 6797	 * into the now unreachable buffer. Somewhat complicated by the
 6798	 * fact that rb::event_lock otherwise nests inside mmap_mutex.
 6799	 */
 6800again:
 6801	rcu_read_lock();
 6802	list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
 6803		if (!atomic_long_inc_not_zero(&event->refcount)) {
 6804			/*
 6805			 * This event is en-route to free_event() which will
 6806			 * detach it and remove it from the list.
 6807			 */
 6808			continue;
 6809		}
 6810		rcu_read_unlock();
 6811
 6812		mutex_lock(&event->mmap_mutex);
 6813		/*
 6814		 * Check we didn't race with perf_event_set_output() which can
 6815		 * swizzle the rb from under us while we were waiting to
 6816		 * acquire mmap_mutex.
 6817		 *
 6818		 * If we find a different rb; ignore this event, a next
 6819		 * iteration will no longer find it on the list. We have to
 6820		 * still restart the iteration to make sure we're not now
 6821		 * iterating the wrong list.
 6822		 */
 6823		if (event->rb == rb)
 6824			ring_buffer_attach(event, NULL);
 6825
 6826		mutex_unlock(&event->mmap_mutex);
 6827		put_event(event);
 6828
 6829		/*
 6830		 * Restart the iteration; either we're on the wrong list or
 6831		 * destroyed its integrity by doing a deletion.
 6832		 */
 6833		goto again;
 6834	}
 6835	rcu_read_unlock();
 6836
 6837	/*
 6838	 * It could be there's still a few 0-ref events on the list; they'll
 6839	 * get cleaned up by free_event() -- they'll also still have their
 6840	 * ref on the rb and will free it whenever they are done with it.
 6841	 *
 6842	 * Aside from that, this buffer is 'fully' detached and unmapped,
 6843	 * undo the VM accounting.
 6844	 */
 6845
 6846	atomic_long_sub((size >> PAGE_SHIFT) + 1 - mmap_locked,
 6847			&mmap_user->locked_vm);
 6848	atomic64_sub(mmap_locked, &vma->vm_mm->pinned_vm);
 6849	free_uid(mmap_user);
 6850
 6851out_put:
 6852	ring_buffer_put(rb); /* could be last */
 6853}
 6854
 6855static vm_fault_t perf_mmap_pfn_mkwrite(struct vm_fault *vmf)
 6856{
 6857	/* The first page is the user control page, others are read-only. */
 6858	return vmf->pgoff == 0 ? 0 : VM_FAULT_SIGBUS;
 6859}
 6860
 6861static int perf_mmap_may_split(struct vm_area_struct *vma, unsigned long addr)
 6862{
 6863	/*
 6864	 * Forbid splitting perf mappings to prevent refcount leaks due to
 6865	 * the resulting non-matching offsets and sizes. See open()/close().
 6866	 */
 6867	return -EINVAL;
 6868}
 6869
 6870static const struct vm_operations_struct perf_mmap_vmops = {
 6871	.open		= perf_mmap_open,
 6872	.close		= perf_mmap_close, /* non mergeable */
 6873	.pfn_mkwrite	= perf_mmap_pfn_mkwrite,
 6874	.may_split	= perf_mmap_may_split,
 6875};
 6876
 6877static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma)
 6878{
 6879	unsigned long nr_pages = vma_pages(vma);
 6880	int err = 0;
 6881	unsigned long pagenum;
 6882
 6883	/*
 6884	 * We map this as a VM_PFNMAP VMA.
 6885	 *
 6886	 * This is not ideal as this is designed broadly for mappings of PFNs
 6887	 * referencing memory-mapped I/O ranges or non-system RAM i.e. for which
 6888	 * !pfn_valid(pfn).
 6889	 *
 6890	 * We are mapping kernel-allocated memory (memory we manage ourselves)
 6891	 * which would more ideally be mapped using vm_insert_page() or a
 6892	 * similar mechanism, that is as a VM_MIXEDMAP mapping.
 6893	 *
 6894	 * However this won't work here, because:
 6895	 *
 6896	 * 1. It uses vma->vm_page_prot, but this field has not been completely
 6897	 *    setup at the point of the f_op->mmp() hook, so we are unable to
 6898	 *    indicate that this should be mapped CoW in order that the
 6899	 *    mkwrite() hook can be invoked to make the first page R/W and the
 6900	 *    rest R/O as desired.
 6901	 *
 6902	 * 2. Anything other than a VM_PFNMAP of valid PFNs will result in
 6903	 *    vm_normal_page() returning a struct page * pointer, which means
 6904	 *    vm_ops->page_mkwrite() will be invoked rather than
 6905	 *    vm_ops->pfn_mkwrite(), and this means we have to set page->mapping
 6906	 *    to work around retry logic in the fault handler, however this
 6907	 *    field is no longer allowed to be used within struct page.
 6908	 *
 6909	 * 3. Having a struct page * made available in the fault logic also
 6910	 *    means that the page gets put on the rmap and becomes
 6911	 *    inappropriately accessible and subject to map and ref counting.
 6912	 *
 6913	 * Ideally we would have a mechanism that could explicitly express our
 6914	 * desires, but this is not currently the case, so we instead use
 6915	 * VM_PFNMAP.
 6916	 *
 6917	 * We manage the lifetime of these mappings with internal refcounts (see
 6918	 * perf_mmap_open() and perf_mmap_close()) so we ensure the lifetime of
 6919	 * this mapping is maintained correctly.
 6920	 */
 6921	for (pagenum = 0; pagenum < nr_pages; pagenum++) {
 6922		unsigned long va = vma->vm_start + PAGE_SIZE * pagenum;
 6923		struct page *page = perf_mmap_to_page(rb, vma->vm_pgoff + pagenum);
 6924
 6925		if (page == NULL) {
 6926			err = -EINVAL;
 6927			break;
 6928		}
 6929
 6930		/* Map readonly, perf_mmap_pfn_mkwrite() called on write fault. */
 6931		err = remap_pfn_range(vma, va, page_to_pfn(page), PAGE_SIZE,
 6932				      vm_get_page_prot(vma->vm_flags & ~VM_SHARED));
 6933		if (err)
 6934			break;
 6935	}
 6936
 6937#ifdef CONFIG_MMU
 6938	/* Clear any partial mappings on error. */
 6939	if (err)
 6940		zap_page_range_single(vma, vma->vm_start, nr_pages * PAGE_SIZE, NULL);
 6941#endif
 6942
 6943	return err;
 6944}
 6945
 6946static bool perf_mmap_calc_limits(struct vm_area_struct *vma, long *user_extra, long *extra)
 6947{
 6948	unsigned long user_locked, user_lock_limit, locked, lock_limit;
 6949	struct user_struct *user = current_user();
 6950
 6951	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
 6952	/* Increase the limit linearly with more CPUs */
 6953	user_lock_limit *= num_online_cpus();
 6954
 6955	user_locked = atomic_long_read(&user->locked_vm);
 6956
 6957	/*
 6958	 * sysctl_perf_event_mlock may have changed, so that
 6959	 *     user->locked_vm > user_lock_limit
 6960	 */
 6961	if (user_locked > user_lock_limit)
 6962		user_locked = user_lock_limit;
 6963	user_locked += *user_extra;
 6964
 6965	if (user_locked > user_lock_limit) {
 6966		/*
 6967		 * charge locked_vm until it hits user_lock_limit;
 6968		 * charge the rest from pinned_vm
 6969		 */
 6970		*extra = user_locked - user_lock_limit;
 6971		*user_extra -= *extra;
 6972	}
 6973
 6974	lock_limit = rlimit(RLIMIT_MEMLOCK);
 6975	lock_limit >>= PAGE_SHIFT;
 6976	locked = atomic64_read(&vma->vm_mm->pinned_vm) + *extra;
 6977
 6978	return locked <= lock_limit || !perf_is_paranoid() || capable(CAP_IPC_LOCK);
 6979}
 6980
 6981static void perf_mmap_account(struct vm_area_struct *vma, long user_extra, long extra)
 6982{
 6983	struct user_struct *user = current_user();
 6984
 6985	atomic_long_add(user_extra, &user->locked_vm);
 6986	atomic64_add(extra, &vma->vm_mm->pinned_vm);
 6987}
 6988
 6989static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event,
 6990			unsigned long nr_pages)
 6991{
 6992	long extra = 0, user_extra = nr_pages;
 6993	struct perf_buffer *rb;
 6994	int rb_flags = 0;
 6995
 6996	nr_pages -= 1;
 6997
 6998	/*
 6999	 * If we have rb pages ensure they're a power-of-two number, so we
 7000	 * can do bitmasks instead of modulo.
 7001	 */
 7002	if (nr_pages != 0 && !is_power_of_2(nr_pages))
 7003		return -EINVAL;
 7004
 7005	WARN_ON_ONCE(event->ctx->parent_ctx);
 7006
 7007	if (event->rb) {
 7008		if (data_page_nr(event->rb) != nr_pages)
 7009			return -EINVAL;
 7010
 7011		/*
 7012		 * If this event doesn't have mmap_count, we're attempting to
 7013		 * create an alias of another event's mmap(); this would mean
 7014		 * both events will end up scribbling the same user_page;
 7015		 * which makes no sense.
 7016		 */
 7017		if (!refcount_read(&event->mmap_count))
 7018			return -EBUSY;
 7019
 7020		if (refcount_inc_not_zero(&event->rb->mmap_count)) {
 7021			/*
 7022			 * Success -- managed to mmap() the same buffer
 7023			 * multiple times.
 7024			 */
 7025			perf_mmap_account(vma, user_extra, extra);
 7026			refcount_inc(&event->mmap_count);
 7027			return 0;
 7028		}
 7029
 7030		/*
 7031		 * Raced against perf_mmap_close()'s
 7032		 * refcount_dec_and_mutex_lock() remove the
 7033		 * event and continue as if !event->rb
 7034		 */
 7035		ring_buffer_attach(event, NULL);
 7036	}
 7037
 7038	if (!perf_mmap_calc_limits(vma, &user_extra, &extra))
 7039		return -EPERM;
 7040
 7041	if (vma->vm_flags & VM_WRITE)
 7042		rb_flags |= RING_BUFFER_WRITABLE;
 7043
 7044	rb = rb_alloc(nr_pages,
 7045		      event->attr.watermark ? event->attr.wakeup_watermark : 0,
 7046		      event->cpu, rb_flags);
 7047
 7048	if (!rb)
 7049		return -ENOMEM;
 7050
 7051	refcount_set(&rb->mmap_count, 1);
 7052	rb->mmap_user = get_current_user();
 7053	rb->mmap_locked = extra;
 7054
 7055	ring_buffer_attach(event, rb);
 7056
 7057	perf_event_update_time(event);
 7058	perf_event_init_userpage(event);
 7059	perf_event_update_userpage(event);
 7060
 7061	perf_mmap_account(vma, user_extra, extra);
 7062	refcount_set(&event->mmap_count, 1);
 7063
 7064	return 0;
 7065}
 7066
 7067static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event,
 7068			 unsigned long nr_pages)
 7069{
 7070	long extra = 0, user_extra = nr_pages;
 7071	u64 aux_offset, aux_size;
 7072	struct perf_buffer *rb;
 7073	int ret, rb_flags = 0;
 7074
 7075	rb = event->rb;
 7076	if (!rb)
 7077		return -EINVAL;
 7078
 7079	guard(mutex)(&rb->aux_mutex);
 7080
 7081	/*
 7082	 * AUX area mapping: if rb->aux_nr_pages != 0, it's already
 7083	 * mapped, all subsequent mappings should have the same size
 7084	 * and offset. Must be above the normal perf buffer.
 7085	 */
 7086	aux_offset = READ_ONCE(rb->user_page->aux_offset);
 7087	aux_size = READ_ONCE(rb->user_page->aux_size);
 7088
 7089	if (aux_offset < perf_data_size(rb) + PAGE_SIZE)
 7090		return -EINVAL;
 7091
 7092	if (aux_offset != vma->vm_pgoff << PAGE_SHIFT)
 7093		return -EINVAL;
 7094
 7095	/* already mapped with a different offset */
 7096	if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff)
 7097		return -EINVAL;
 7098
 7099	if (aux_size != nr_pages * PAGE_SIZE)
 7100		return -EINVAL;
 7101
 7102	/* already mapped with a different size */
 7103	if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)
 7104		return -EINVAL;
 7105
 7106	if (!is_power_of_2(nr_pages))
 7107		return -EINVAL;
 7108
 7109	if (!refcount_inc_not_zero(&rb->mmap_count))
 7110		return -EINVAL;
 7111
 7112	if (rb_has_aux(rb)) {
 7113		refcount_inc(&rb->aux_mmap_count);
 7114
 7115	} else {
 7116		if (!perf_mmap_calc_limits(vma, &user_extra, &extra)) {
 7117			refcount_dec(&rb->mmap_count);
 7118			return -EPERM;
 7119		}
 7120
 7121		WARN_ON(!rb && event->rb);
 7122
 7123		if (vma->vm_flags & VM_WRITE)
 7124			rb_flags |= RING_BUFFER_WRITABLE;
 7125
 7126		ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,
 7127				   event->attr.aux_watermark, rb_flags);
 7128		if (ret) {
 7129			refcount_dec(&rb->mmap_count);
 7130			return ret;
 7131		}
 7132
 7133		refcount_set(&rb->aux_mmap_count, 1);
 7134		rb->aux_mmap_locked = extra;
 7135	}
 7136
 7137	perf_mmap_account(vma, user_extra, extra);
 7138	refcount_inc(&event->mmap_count);
 7139
 7140	return 0;
 7141}
 7142
 7143static int perf_mmap(struct file *file, struct vm_area_struct *vma)
 7144{
 7145	struct perf_event *event = file->private_data;
 7146	unsigned long vma_size, nr_pages;
 7147	mapped_f mapped;
 7148	int ret;
 7149
 7150	/*
 7151	 * Don't allow mmap() of inherited per-task counters. This would
 7152	 * create a performance issue due to all children writing to the
 7153	 * same rb.
 7154	 */
 7155	if (event->cpu == -1 && event->attr.inherit)
 7156		return -EINVAL;
 7157
 7158	if (!(vma->vm_flags & VM_SHARED))
 7159		return -EINVAL;
 7160
 7161	ret = security_perf_event_read(event);
 7162	if (ret)
 7163		return ret;
 7164
 7165	vma_size = vma->vm_end - vma->vm_start;
 7166	nr_pages = vma_size / PAGE_SIZE;
 7167
 7168	if (nr_pages > INT_MAX)
 7169		return -ENOMEM;
 7170
 7171	if (vma_size != PAGE_SIZE * nr_pages)
 7172		return -EINVAL;
 7173
 7174	scoped_guard (mutex, &event->mmap_mutex) {
 7175		/*
 7176		 * This relies on __pmu_detach_event() taking mmap_mutex after marking
 7177		 * the event REVOKED. Either we observe the state, or __pmu_detach_event()
 7178		 * will detach the rb created here.
 7179		 */
 7180		if (event->state <= PERF_EVENT_STATE_REVOKED)
 7181			return -ENODEV;
 7182
 7183		if (vma->vm_pgoff == 0)
 7184			ret = perf_mmap_rb(vma, event, nr_pages);
 7185		else
 7186			ret = perf_mmap_aux(vma, event, nr_pages);
 7187		if (ret)
 7188			return ret;
 7189
 7190		/*
 7191		 * Since pinned accounting is per vm we cannot allow fork() to copy our
 7192		 * vma.
 7193		 */
 7194		vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP);
 7195		vma->vm_ops = &perf_mmap_vmops;
 7196
 7197		mapped = get_mapped(event, event_mapped);
 7198		if (mapped)
 7199			mapped(event, vma->vm_mm);
 7200
 7201		/*
 7202		 * Try to map it into the page table. On fail, invoke
 7203		 * perf_mmap_close() to undo the above, as the callsite expects
 7204		 * full cleanup in this case and therefore does not invoke
 7205		 * vmops::close().
 7206		 */
 7207		ret = map_range(event->rb, vma);
 7208		if (ret)
 7209			perf_mmap_close(vma);
 7210	}
 7211
 7212	return ret;
 7213}
 7214
 7215static int perf_fasync(int fd, struct file *filp, int on)
 7216{
 7217	struct inode *inode = file_inode(filp);
 7218	struct perf_event *event = filp->private_data;
 7219	int retval;
 7220
 7221	if (event->state <= PERF_EVENT_STATE_REVOKED)
 7222		return -ENODEV;
 7223
 7224	inode_lock(inode);
 7225	retval = fasync_helper(fd, filp, on, &event->fasync);
 7226	inode_unlock(inode);
 7227
 7228	if (retval < 0)
 7229		return retval;
 7230
 7231	return 0;
 7232}
 7233
 7234static const struct file_operations perf_fops = {
 7235	.release		= perf_release,
 7236	.read			= perf_read,
 7237	.poll			= perf_poll,
 7238	.unlocked_ioctl		= perf_ioctl,
 7239	.compat_ioctl		= perf_compat_ioctl,
 7240	.mmap			= perf_mmap,
 7241	.fasync			= perf_fasync,
 7242};
 7243
 7244/*
 7245 * Perf event wakeup
 7246 *
 7247 * If there's data, ensure we set the poll() state and publish everything
 7248 * to user-space before waking everybody up.
 7249 */
 7250
 7251void perf_event_wakeup(struct perf_event *event)
 7252{
 7253	ring_buffer_wakeup(event);
 7254
 7255	if (event->pending_kill) {
 7256		kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill);
 7257		event->pending_kill = 0;
 7258	}
 7259}
 7260
 7261static void perf_sigtrap(struct perf_event *event)
 7262{
 7263	/*
 7264	 * Both perf_pending_task() and perf_pending_irq() can race with the
 7265	 * task exiting.
 7266	 */
 7267	if (current->flags & PF_EXITING)
 7268		return;
 7269
 7270	/*
 7271	 * We'd expect this to only occur if the irq_work is delayed and either
 7272	 * ctx->task or current has changed in the meantime. This can be the
 7273	 * case on architectures that do not implement arch_irq_work_raise().
 7274	 */
 7275	if (WARN_ON_ONCE(event->ctx->task != current))
 7276		return;
 7277
 7278	send_sig_perf((void __user *)event->pending_addr,
 7279		      event->orig_type, event->attr.sig_data);
 7280}
 7281
 7282/*
 7283 * Deliver the pending work in-event-context or follow the context.
 7284 */
 7285static void __perf_pending_disable(struct perf_event *event)
 7286{
 7287	int cpu = READ_ONCE(event->oncpu);
 7288
 7289	/*
 7290	 * If the event isn't running; we done. event_sched_out() will have
 7291	 * taken care of things.
 7292	 */
 7293	if (cpu < 0)
 7294		return;
 7295
 7296	/*
 7297	 * Yay, we hit home and are in the context of the event.
 7298	 */
 7299	if (cpu == smp_processor_id()) {
 7300		if (event->pending_disable) {
 7301			event->pending_disable = 0;
 7302			perf_event_disable_local(event);
 7303		}
 7304		return;
 7305	}
 7306
 7307	/*
 7308	 *  CPU-A			CPU-B
 7309	 *
 7310	 *  perf_event_disable_inatomic()
 7311	 *    @pending_disable = 1;
 7312	 *    irq_work_queue();
 7313	 *
 7314	 *  sched-out
 7315	 *    @pending_disable = 0;
 7316	 *
 7317	 *				sched-in
 7318	 *				perf_event_disable_inatomic()
 7319	 *				  @pending_disable = 1;
 7320	 *				  irq_work_queue(); // FAILS
 7321	 *
 7322	 *  irq_work_run()
 7323	 *    perf_pending_disable()
 7324	 *
 7325	 * But the event runs on CPU-B and wants disabling there.
 7326	 */
 7327	irq_work_queue_on(&event->pending_disable_irq, cpu);
 7328}
 7329
 7330static void perf_pending_disable(struct irq_work *entry)
 7331{
 7332	struct perf_event *event = container_of(entry, struct perf_event, pending_disable_irq);
 7333	int rctx;
 7334
 7335	/*
 7336	 * If we 'fail' here, that's OK, it means recursion is already disabled
 7337	 * and we won't recurse 'further'.
 7338	 */
 7339	rctx = perf_swevent_get_recursion_context();
 7340	__perf_pending_disable(event);
 7341	if (rctx >= 0)
 7342		perf_swevent_put_recursion_context(rctx);
 7343}
 7344
 7345static void perf_pending_irq(struct irq_work *entry)
 7346{
 7347	struct perf_event *event = container_of(entry, struct perf_event, pending_irq);
 7348	int rctx;
 7349
 7350	/*
 7351	 * If we 'fail' here, that's OK, it means recursion is already disabled
 7352	 * and we won't recurse 'further'.
 7353	 */
 7354	rctx = perf_swevent_get_recursion_context();
 7355
 7356	/*
 7357	 * The wakeup isn't bound to the context of the event -- it can happen
 7358	 * irrespective of where the event is.
 7359	 */
 7360	if (event->pending_wakeup) {
 7361		event->pending_wakeup = 0;
 7362		perf_event_wakeup(event);
 7363	}
 7364
 7365	if (rctx >= 0)
 7366		perf_swevent_put_recursion_context(rctx);
 7367}
 7368
 7369static void perf_pending_task(struct callback_head *head)
 7370{
 7371	struct perf_event *event = container_of(head, struct perf_event, pending_task);
 7372	int rctx;
 7373
 7374	/*
 7375	 * If we 'fail' here, that's OK, it means recursion is already disabled
 7376	 * and we won't recurse 'further'.
 7377	 */
 7378	rctx = perf_swevent_get_recursion_context();
 7379
 7380	if (event->pending_work) {
 7381		event->pending_work = 0;
 7382		perf_sigtrap(event);
 7383		local_dec(&event->ctx->nr_no_switch_fast);
 7384	}
 7385	put_event(event);
 7386
 7387	if (rctx >= 0)
 7388		perf_swevent_put_recursion_context(rctx);
 7389}
 7390
 7391#ifdef CONFIG_GUEST_PERF_EVENTS
 7392struct perf_guest_info_callbacks __rcu *perf_guest_cbs;
 7393
 7394DEFINE_STATIC_CALL_RET0(__perf_guest_state, *perf_guest_cbs->state);
 7395DEFINE_STATIC_CALL_RET0(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
 7396DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
 7397
 7398void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
 7399{
 7400	if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs)))
 7401		return;
 7402
 7403	rcu_assign_pointer(perf_guest_cbs, cbs);
 7404	static_call_update(__perf_guest_state, cbs->state);
 7405	static_call_update(__perf_guest_get_ip, cbs->get_ip);
 7406
 7407	/* Implementing ->handle_intel_pt_intr is optional. */
 7408	if (cbs->handle_intel_pt_intr)
 7409		static_call_update(__perf_guest_handle_intel_pt_intr,
 7410				   cbs->handle_intel_pt_intr);
 7411}
 7412EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
 7413
 7414void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
 7415{
 7416	if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs) != cbs))
 7417		return;
 7418
 7419	rcu_assign_pointer(perf_guest_cbs, NULL);
 7420	static_call_update(__perf_guest_state, (void *)&__static_call_return0);
 7421	static_call_update(__perf_guest_get_ip, (void *)&__static_call_return0);
 7422	static_call_update(__perf_guest_handle_intel_pt_intr,
 7423			   (void *)&__static_call_return0);
 7424	synchronize_rcu();
 7425}
 7426EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
 7427#endif
 7428
 7429static bool should_sample_guest(struct perf_event *event)
 7430{
 7431	return !event->attr.exclude_guest && perf_guest_state();
 7432}
 7433
 7434unsigned long perf_misc_flags(struct perf_event *event,
 7435			      struct pt_regs *regs)
 7436{
 7437	if (should_sample_guest(event))
 7438		return perf_arch_guest_misc_flags(regs);
 7439
 7440	return perf_arch_misc_flags(regs);
 7441}
 7442
 7443unsigned long perf_instruction_pointer(struct perf_event *event,
 7444				       struct pt_regs *regs)
 7445{
 7446	if (should_sample_guest(event))
 7447		return perf_guest_get_ip();
 7448
 7449	return perf_arch_instruction_pointer(regs);
 7450}
 7451
 7452static void
 7453perf_output_sample_regs(struct perf_output_handle *handle,
 7454			struct pt_regs *regs, u64 mask)
 7455{
 7456	int bit;
 7457	DECLARE_BITMAP(_mask, 64);
 7458
 7459	bitmap_from_u64(_mask, mask);
 7460	for_each_set_bit(bit, _mask, sizeof(mask) * BITS_PER_BYTE) {
 7461		u64 val;
 7462
 7463		val = perf_reg_value(regs, bit);
 7464		perf_output_put(handle, val);
 7465	}
 7466}
 7467
 7468static void perf_sample_regs_user(struct perf_regs *regs_user,
 7469				  struct pt_regs *regs)
 7470{
 7471	if (user_mode(regs)) {
 7472		regs_user->abi = perf_reg_abi(current);
 7473		regs_user->regs = regs;
 7474	} else if (is_user_task(current)) {
 7475		perf_get_regs_user(regs_user, regs);
 7476	} else {
 7477		regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
 7478		regs_user->regs = NULL;
 7479	}
 7480}
 7481
 7482static void perf_sample_regs_intr(struct perf_regs *regs_intr,
 7483				  struct pt_regs *regs)
 7484{
 7485	regs_intr->regs = regs;
 7486	regs_intr->abi  = perf_reg_abi(current);
 7487}
 7488
 7489
 7490/*
 7491 * Get remaining task size from user stack pointer.
 7492 *
 7493 * It'd be better to take stack vma map and limit this more
 7494 * precisely, but there's no way to get it safely under interrupt,
 7495 * so using TASK_SIZE as limit.
 7496 */
 7497static u64 perf_ustack_task_size(struct pt_regs *regs)
 7498{
 7499	unsigned long addr = perf_user_stack_pointer(regs);
 7500
 7501	if (!addr || addr >= TASK_SIZE)
 7502		return 0;
 7503
 7504	return TASK_SIZE - addr;
 7505}
 7506
 7507static u16
 7508perf_sample_ustack_size(u16 stack_size, u16 header_size,
 7509			struct pt_regs *regs)
 7510{
 7511	u64 task_size;
 7512
 7513	/* No regs, no stack pointer, no dump. */
 7514	if (!regs)
 7515		return 0;
 7516
 7517	/* No mm, no stack, no dump. */
 7518	if (!current->mm)
 7519		return 0;
 7520
 7521	/*
 7522	 * Check if we fit in with the requested stack size into the:
 7523	 * - TASK_SIZE
 7524	 *   If we don't, we limit the size to the TASK_SIZE.
 7525	 *
 7526	 * - remaining sample size
 7527	 *   If we don't, we customize the stack size to
 7528	 *   fit in to the remaining sample size.
 7529	 */
 7530
 7531	task_size  = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
 7532	stack_size = min(stack_size, (u16) task_size);
 7533
 7534	/* Current header size plus static size and dynamic size. */
 7535	header_size += 2 * sizeof(u64);
 7536
 7537	/* Do we fit in with the current stack dump size? */
 7538	if ((u16) (header_size + stack_size) < header_size) {
 7539		/*
 7540		 * If we overflow the maximum size for the sample,
 7541		 * we customize the stack dump size to fit in.
 7542		 */
 7543		stack_size = USHRT_MAX - header_size - sizeof(u64);
 7544		stack_size = round_up(stack_size, sizeof(u64));
 7545	}
 7546
 7547	return stack_size;
 7548}
 7549
 7550static void
 7551perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
 7552			  struct pt_regs *regs)
 7553{
 7554	/* Case of a kernel thread, nothing to dump */
 7555	if (!regs) {
 7556		u64 size = 0;
 7557		perf_output_put(handle, size);
 7558	} else {
 7559		unsigned long sp;
 7560		unsigned int rem;
 7561		u64 dyn_size;
 7562
 7563		/*
 7564		 * We dump:
 7565		 * static size
 7566		 *   - the size requested by user or the best one we can fit
 7567		 *     in to the sample max size
 7568		 * data
 7569		 *   - user stack dump data
 7570		 * dynamic size
 7571		 *   - the actual dumped size
 7572		 */
 7573
 7574		/* Static size. */
 7575		perf_output_put(handle, dump_size);
 7576
 7577		/* Data. */
 7578		sp = perf_user_stack_pointer(regs);
 7579		rem = __output_copy_user(handle, (void *) sp, dump_size);
 7580		dyn_size = dump_size - rem;
 7581
 7582		perf_output_skip(handle, rem);
 7583
 7584		/* Dynamic size. */
 7585		perf_output_put(handle, dyn_size);
 7586	}
 7587}
 7588
 7589static unsigned long perf_prepare_sample_aux(struct perf_event *event,
 7590					  struct perf_sample_data *data,
 7591					  size_t size)
 7592{
 7593	struct perf_event *sampler = event->aux_event;
 7594	struct perf_buffer *rb;
 7595
 7596	data->aux_size = 0;
 7597
 7598	if (!sampler)
 7599		goto out;
 7600
 7601	if (WARN_ON_ONCE(READ_ONCE(sampler->state) != PERF_EVENT_STATE_ACTIVE))
 7602		goto out;
 7603
 7604	if (WARN_ON_ONCE(READ_ONCE(sampler->oncpu) != smp_processor_id()))
 7605		goto out;
 7606
 7607	rb = ring_buffer_get(sampler);
 7608	if (!rb)
 7609		goto out;
 7610
 7611	/*
 7612	 * If this is an NMI hit inside sampling code, don't take
 7613	 * the sample. See also perf_aux_sample_output().
 7614	 */
 7615	if (READ_ONCE(rb->aux_in_sampling)) {
 7616		data->aux_size = 0;
 7617	} else {
 7618		size = min_t(size_t, size, perf_aux_size(rb));
 7619		data->aux_size = ALIGN(size, sizeof(u64));
 7620	}
 7621	ring_buffer_put(rb);
 7622
 7623out:
 7624	return data->aux_size;
 7625}
 7626
 7627static long perf_pmu_snapshot_aux(struct perf_buffer *rb,
 7628                                 struct perf_event *event,
 7629                                 struct perf_output_handle *handle,
 7630                                 unsigned long size)
 7631{
 7632	unsigned long flags;
 7633	long ret;
 7634
 7635	/*
 7636	 * Normal ->start()/->stop() callbacks run in IRQ mode in scheduler
 7637	 * paths. If we start calling them in NMI context, they may race with
 7638	 * the IRQ ones, that is, for example, re-starting an event that's just
 7639	 * been stopped, which is why we're using a separate callback that
 7640	 * doesn't change the event state.
 7641	 *
 7642	 * IRQs need to be disabled to prevent IPIs from racing with us.
 7643	 */
 7644	local_irq_save(flags);
 7645	/*
 7646	 * Guard against NMI hits inside the critical section;
 7647	 * see also perf_prepare_sample_aux().
 7648	 */
 7649	WRITE_ONCE(rb->aux_in_sampling, 1);
 7650	barrier();
 7651
 7652	ret = event->pmu->snapshot_aux(event, handle, size);
 7653
 7654	barrier();
 7655	WRITE_ONCE(rb->aux_in_sampling, 0);
 7656	local_irq_restore(flags);
 7657
 7658	return ret;
 7659}
 7660
 7661static void perf_aux_sample_output(struct perf_event *event,
 7662				   struct perf_output_handle *handle,
 7663				   struct perf_sample_data *data)
 7664{
 7665	struct perf_event *sampler = event->aux_event;
 7666	struct perf_buffer *rb;
 7667	unsigned long pad;
 7668	long size;
 7669
 7670	if (WARN_ON_ONCE(!sampler || !data->aux_size))
 7671		return;
 7672
 7673	rb = ring_buffer_get(sampler);
 7674	if (!rb)
 7675		return;
 7676
 7677	size = perf_pmu_snapshot_aux(rb, sampler, handle, data->aux_size);
 7678
 7679	/*
 7680	 * An error here means that perf_output_copy() failed (returned a
 7681	 * non-zero surplus that it didn't copy), which in its current
 7682	 * enlightened implementation is not possible. If that changes, we'd
 7683	 * like to know.
 7684	 */
 7685	if (WARN_ON_ONCE(size < 0))
 7686		goto out_put;
 7687
 7688	/*
 7689	 * The pad comes from ALIGN()ing data->aux_size up to u64 in
 7690	 * perf_prepare_sample_aux(), so should not be more than that.
 7691	 */
 7692	pad = data->aux_size - size;
 7693	if (WARN_ON_ONCE(pad >= sizeof(u64)))
 7694		pad = 8;
 7695
 7696	if (pad) {
 7697		u64 zero = 0;
 7698		perf_output_copy(handle, &zero, pad);
 7699	}
 7700
 7701out_put:
 7702	ring_buffer_put(rb);
 7703}
 7704
 7705/*
 7706 * A set of common sample data types saved even for non-sample records
 7707 * when event->attr.sample_id_all is set.
 7708 */
 7709#define PERF_SAMPLE_ID_ALL  (PERF_SAMPLE_TID | PERF_SAMPLE_TIME |	\
 7710			     PERF_SAMPLE_ID | PERF_SAMPLE_STREAM_ID |	\
 7711			     PERF_SAMPLE_CPU | PERF_SAMPLE_IDENTIFIER)
 7712
 7713static void __perf_event_header__init_id(struct perf_sample_data *data,
 7714					 struct perf_event *event,
 7715					 u64 sample_type)
 7716{
 7717	data->type = event->attr.sample_type;
 7718	data->sample_flags |= data->type & PERF_SAMPLE_ID_ALL;
 7719
 7720	if (sample_type & PERF_SAMPLE_TID) {
 7721		/* namespace issues */
 7722		data->tid_entry.pid = perf_event_pid(event, current);
 7723		data->tid_entry.tid = perf_event_tid(event, current);
 7724	}
 7725
 7726	if (sample_type & PERF_SAMPLE_TIME)
 7727		data->time = perf_event_clock(event);
 7728
 7729	if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
 7730		data->id = primary_event_id(event);
 7731
 7732	if (sample_type & PERF_SAMPLE_STREAM_ID)
 7733		data->stream_id = event->id;
 7734
 7735	if (sample_type & PERF_SAMPLE_CPU) {
 7736		data->cpu_entry.cpu	 = raw_smp_processor_id();
 7737		data->cpu_entry.reserved = 0;
 7738	}
 7739}
 7740
 7741void perf_event_header__init_id(struct perf_event_header *header,
 7742				struct perf_sample_data *data,
 7743				struct perf_event *event)
 7744{
 7745	if (event->attr.sample_id_all) {
 7746		header->size += event->id_header_size;
 7747		__perf_event_header__init_id(data, event, event->attr.sample_type);
 7748	}
 7749}
 7750
 7751static void __perf_event__output_id_sample(struct perf_output_handle *handle,
 7752					   struct perf_sample_data *data)
 7753{
 7754	u64 sample_type = data->type;
 7755
 7756	if (sample_type & PERF_SAMPLE_TID)
 7757		perf_output_put(handle, data->tid_entry);
 7758
 7759	if (sample_type & PERF_SAMPLE_TIME)
 7760		perf_output_put(handle, data->time);
 7761
 7762	if (sample_type & PERF_SAMPLE_ID)
 7763		perf_output_put(handle, data->id);
 7764
 7765	if (sample_type & PERF_SAMPLE_STREAM_ID)
 7766		perf_output_put(handle, data->stream_id);
 7767
 7768	if (sample_type & PERF_SAMPLE_CPU)
 7769		perf_output_put(handle, data->cpu_entry);
 7770
 7771	if (sample_type & PERF_SAMPLE_IDENTIFIER)
 7772		perf_output_put(handle, data->id);
 7773}
 7774
 7775void perf_event__output_id_sample(struct perf_event *event,
 7776				  struct perf_output_handle *handle,
 7777				  struct perf_sample_data *sample)
 7778{
 7779	if (event->attr.sample_id_all)
 7780		__perf_event__output_id_sample(handle, sample);
 7781}
 7782
 7783static void perf_output_read_one(struct perf_output_handle *handle,
 7784				 struct perf_event *event,
 7785				 u64 enabled, u64 running)
 7786{
 7787	u64 read_format = event->attr.read_format;
 7788	u64 values[5];
 7789	int n = 0;
 7790
 7791	values[n++] = perf_event_count(event, has_inherit_and_sample_read(&event->attr));
 7792	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
 7793		values[n++] = enabled +
 7794			atomic64_read(&event->child_total_time_enabled);
 7795	}
 7796	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
 7797		values[n++] = running +
 7798			atomic64_read(&event->child_total_time_running);
 7799	}
 7800	if (read_format & PERF_FORMAT_ID)
 7801		values[n++] = primary_event_id(event);
 7802	if (read_format & PERF_FORMAT_LOST)
 7803		values[n++] = atomic64_read(&event->lost_samples);
 7804
 7805	__output_copy(handle, values, n * sizeof(u64));
 7806}
 7807
 7808static void perf_output_read_group(struct perf_output_handle *handle,
 7809				   struct perf_event *event,
 7810				   u64 enabled, u64 running)
 7811{
 7812	struct perf_event *leader = event->group_leader, *sub;
 7813	u64 read_format = event->attr.read_format;
 7814	unsigned long flags;
 7815	u64 values[6];
 7816	int n = 0;
 7817	bool self = has_inherit_and_sample_read(&event->attr);
 7818
 7819	/*
 7820	 * Disabling interrupts avoids all counter scheduling
 7821	 * (context switches, timer based rotation and IPIs).
 7822	 */
 7823	local_irq_save(flags);
 7824
 7825	values[n++] = 1 + leader->nr_siblings;
 7826
 7827	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
 7828		values[n++] = enabled;
 7829
 7830	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
 7831		values[n++] = running;
 7832
 7833	if ((leader != event) && !handle->skip_read)
 7834		perf_pmu_read(leader);
 7835
 7836	values[n++] = perf_event_count(leader, self);
 7837	if (read_format & PERF_FORMAT_ID)
 7838		values[n++] = primary_event_id(leader);
 7839	if (read_format & PERF_FORMAT_LOST)
 7840		values[n++] = atomic64_read(&leader->lost_samples);
 7841
 7842	__output_copy(handle, values, n * sizeof(u64));
 7843
 7844	for_each_sibling_event(sub, leader) {
 7845		n = 0;
 7846
 7847		if ((sub != event) && !handle->skip_read)
 7848			perf_pmu_read(sub);
 7849
 7850		values[n++] = perf_event_count(sub, self);
 7851		if (read_format & PERF_FORMAT_ID)
 7852			values[n++] = primary_event_id(sub);
 7853		if (read_format & PERF_FORMAT_LOST)
 7854			values[n++] = atomic64_read(&sub->lost_samples);
 7855
 7856		__output_copy(handle, values, n * sizeof(u64));
 7857	}
 7858
 7859	local_irq_restore(flags);
 7860}
 7861
 7862#define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
 7863				 PERF_FORMAT_TOTAL_TIME_RUNNING)
 7864
 7865/*
 7866 * XXX PERF_SAMPLE_READ vs inherited events seems difficult.
 7867 *
 7868 * The problem is that its both hard and excessively expensive to iterate the
 7869 * child list, not to mention that its impossible to IPI the children running
 7870 * on another CPU, from interrupt/NMI context.
 7871 *
 7872 * Instead the combination of PERF_SAMPLE_READ and inherit will track per-thread
 7873 * counts rather than attempting to accumulate some value across all children on
 7874 * all cores.
 7875 */
 7876static void perf_output_read(struct perf_output_handle *handle,
 7877			     struct perf_event *event)
 7878{
 7879	u64 enabled = 0, running = 0, now;
 7880	u64 read_format = event->attr.read_format;
 7881
 7882	/*
 7883	 * compute total_time_enabled, total_time_running
 7884	 * based on snapshot values taken when the event
 7885	 * was last scheduled in.
 7886	 *
 7887	 * we cannot simply called update_context_time()
 7888	 * because of locking issue as we are called in
 7889	 * NMI context
 7890	 */
 7891	if (read_format & PERF_FORMAT_TOTAL_TIMES)
 7892		calc_timer_values(event, &now, &enabled, &running);
 7893
 7894	if (event->attr.read_format & PERF_FORMAT_GROUP)
 7895		perf_output_read_group(handle, event, enabled, running);
 7896	else
 7897		perf_output_read_one(handle, event, enabled, running);
 7898}
 7899
 7900void perf_output_sample(struct perf_output_handle *handle,
 7901			struct perf_event_header *header,
 7902			struct perf_sample_data *data,
 7903			struct perf_event *event)
 7904{
 7905	u64 sample_type = data->type;
 7906
 7907	if (data->sample_flags & PERF_SAMPLE_READ)
 7908		handle->skip_read = 1;
 7909
 7910	perf_output_put(handle, *header);
 7911
 7912	if (sample_type & PERF_SAMPLE_IDENTIFIER)
 7913		perf_output_put(handle, data->id);
 7914
 7915	if (sample_type & PERF_SAMPLE_IP)
 7916		perf_output_put(handle, data->ip);
 7917
 7918	if (sample_type & PERF_SAMPLE_TID)
 7919		perf_output_put(handle, data->tid_entry);
 7920
 7921	if (sample_type & PERF_SAMPLE_TIME)
 7922		perf_output_put(handle, data->time);
 7923
 7924	if (sample_type & PERF_SAMPLE_ADDR)
 7925		perf_output_put(handle, data->addr);
 7926
 7927	if (sample_type & PERF_SAMPLE_ID)
 7928		perf_output_put(handle, data->id);
 7929
 7930	if (sample_type & PERF_SAMPLE_STREAM_ID)
 7931		perf_output_put(handle, data->stream_id);
 7932
 7933	if (sample_type & PERF_SAMPLE_CPU)
 7934		perf_output_put(handle, data->cpu_entry);
 7935
 7936	if (sample_type & PERF_SAMPLE_PERIOD)
 7937		perf_output_put(handle, data->period);
 7938
 7939	if (sample_type & PERF_SAMPLE_READ)
 7940		perf_output_read(handle, event);
 7941
 7942	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
 7943		int size = 1;
 7944
 7945		size += data->callchain->nr;
 7946		size *= sizeof(u64);
 7947		__output_copy(handle, data->callchain, size);
 7948	}
 7949
 7950	if (sample_type & PERF_SAMPLE_RAW) {
 7951		struct perf_raw_record *raw = data->raw;
 7952
 7953		if (raw) {
 7954			struct perf_raw_frag *frag = &raw->frag;
 7955
 7956			perf_output_put(handle, raw->size);
 7957			do {
 7958				if (frag->copy) {
 7959					__output_custom(handle, frag->copy,
 7960							frag->data, frag->size);
 7961				} else {
 7962					__output_copy(handle, frag->data,
 7963						      frag->size);
 7964				}
 7965				if (perf_raw_frag_last(frag))
 7966					break;
 7967				frag = frag->next;
 7968			} while (1);
 7969			if (frag->pad)
 7970				__output_skip(handle, NULL, frag->pad);
 7971		} else {
 7972			struct {
 7973				u32	size;
 7974				u32	data;
 7975			} raw = {
 7976				.size = sizeof(u32),
 7977				.data = 0,
 7978			};
 7979			perf_output_put(handle, raw);
 7980		}
 7981	}
 7982
 7983	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
 7984		if (data->br_stack) {
 7985			size_t size;
 7986
 7987			size = data->br_stack->nr
 7988			     * sizeof(struct perf_branch_entry);
 7989
 7990			perf_output_put(handle, data->br_stack->nr);
 7991			if (branch_sample_hw_index(event))
 7992				perf_output_put(handle, data->br_stack->hw_idx);
 7993			perf_output_copy(handle, data->br_stack->entries, size);
 7994			/*
 7995			 * Add the extension space which is appended
 7996			 * right after the struct perf_branch_stack.
 7997			 */
 7998			if (data->br_stack_cntr) {
 7999				size = data->br_stack->nr * sizeof(u64);
 8000				perf_output_copy(handle, data->br_stack_cntr, size);
 8001			}
 8002		} else {
 8003			/*
 8004			 * we always store at least the value of nr
 8005			 */
 8006			u64 nr = 0;
 8007			perf_output_put(handle, nr);
 8008		}
 8009	}
 8010
 8011	if (sample_type & PERF_SAMPLE_REGS_USER) {
 8012		u64 abi = data->regs_user.abi;
 8013
 8014		/*
 8015		 * If there are no regs to dump, notice it through
 8016		 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
 8017		 */
 8018		perf_output_put(handle, abi);
 8019
 8020		if (abi) {
 8021			u64 mask = event->attr.sample_regs_user;
 8022			perf_output_sample_regs(handle,
 8023						data->regs_user.regs,
 8024						mask);
 8025		}
 8026	}
 8027
 8028	if (sample_type & PERF_SAMPLE_STACK_USER) {
 8029		perf_output_sample_ustack(handle,
 8030					  data->stack_user_size,
 8031					  data->regs_user.regs);
 8032	}
 8033
 8034	if (sample_type & PERF_SAMPLE_WEIGHT_TYPE)
 8035		perf_output_put(handle, data->weight.full);
 8036
 8037	if (sample_type & PERF_SAMPLE_DATA_SRC)
 8038		perf_output_put(handle, data->data_src.val);
 8039
 8040	if (sample_type & PERF_SAMPLE_TRANSACTION)
 8041		perf_output_put(handle, data->txn);
 8042
 8043	if (sample_type & PERF_SAMPLE_REGS_INTR) {
 8044		u64 abi = data->regs_intr.abi;
 8045		/*
 8046		 * If there are no regs to dump, notice it through
 8047		 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
 8048		 */
 8049		perf_output_put(handle, abi);
 8050
 8051		if (abi) {
 8052			u64 mask = event->attr.sample_regs_intr;
 8053
 8054			perf_output_sample_regs(handle,
 8055						data->regs_intr.regs,
 8056						mask);
 8057		}
 8058	}
 8059
 8060	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
 8061		perf_output_put(handle, data->phys_addr);
 8062
 8063	if (sample_type & PERF_SAMPLE_CGROUP)
 8064		perf_output_put(handle, data->cgroup);
 8065
 8066	if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)
 8067		perf_output_put(handle, data->data_page_size);
 8068
 8069	if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)
 8070		perf_output_put(handle, data->code_page_size);
 8071
 8072	if (sample_type & PERF_SAMPLE_AUX) {
 8073		perf_output_put(handle, data->aux_size);
 8074
 8075		if (data->aux_size)
 8076			perf_aux_sample_output(event, handle, data);
 8077	}
 8078
 8079	if (!event->attr.watermark) {
 8080		int wakeup_events = event->attr.wakeup_events;
 8081
 8082		if (wakeup_events) {
 8083			struct perf_buffer *rb = handle->rb;
 8084			int events = local_inc_return(&rb->events);
 8085
 8086			if (events >= wakeup_events) {
 8087				local_sub(wakeup_events, &rb->events);
 8088				local_inc(&rb->wakeup);
 8089			}
 8090		}
 8091	}
 8092}
 8093
 8094static u64 perf_virt_to_phys(u64 virt)
 8095{
 8096	u64 phys_addr = 0;
 8097
 8098	if (!virt)
 8099		return 0;
 8100
 8101	if (virt >= TASK_SIZE) {
 8102		/* If it's vmalloc()d memory, leave phys_addr as 0 */
 8103		if (virt_addr_valid((void *)(uintptr_t)virt) &&
 8104		    !(virt >= VMALLOC_START && virt < VMALLOC_END))
 8105			phys_addr = (u64)virt_to_phys((void *)(uintptr_t)virt);
 8106	} else {
 8107		/*
 8108		 * Walking the pages tables for user address.
 8109		 * Interrupts are disabled, so it prevents any tear down
 8110		 * of the page tables.
 8111		 * Try IRQ-safe get_user_page_fast_only first.
 8112		 * If failed, leave phys_addr as 0.
 8113		 */
 8114		if (is_user_task(current)) {
 8115			struct page *p;
 8116
 8117			pagefault_disable();
 8118			if (get_user_page_fast_only(virt, 0, &p)) {
 8119				phys_addr = page_to_phys(p) + virt % PAGE_SIZE;
 8120				put_page(p);
 8121			}
 8122			pagefault_enable();
 8123		}
 8124	}
 8125
 8126	return phys_addr;
 8127}
 8128
 8129/*
 8130 * Return the pagetable size of a given virtual address.
 8131 */
 8132static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr)
 8133{
 8134	u64 size = 0;
 8135
 8136#ifdef CONFIG_HAVE_GUP_FAST
 8137	pgd_t *pgdp, pgd;
 8138	p4d_t *p4dp, p4d;
 8139	pud_t *pudp, pud;
 8140	pmd_t *pmdp, pmd;
 8141	pte_t *ptep, pte;
 8142
 8143	pgdp = pgd_offset(mm, addr);
 8144	pgd = READ_ONCE(*pgdp);
 8145	if (pgd_none(pgd))
 8146		return 0;
 8147
 8148	if (pgd_leaf(pgd))
 8149		return pgd_leaf_size(pgd);
 8150
 8151	p4dp = p4d_offset_lockless(pgdp, pgd, addr);
 8152	p4d = READ_ONCE(*p4dp);
 8153	if (!p4d_present(p4d))
 8154		return 0;
 8155
 8156	if (p4d_leaf(p4d))
 8157		return p4d_leaf_size(p4d);
 8158
 8159	pudp = pud_offset_lockless(p4dp, p4d, addr);
 8160	pud = READ_ONCE(*pudp);
 8161	if (!pud_present(pud))
 8162		return 0;
 8163
 8164	if (pud_leaf(pud))
 8165		return pud_leaf_size(pud);
 8166
 8167	pmdp = pmd_offset_lockless(pudp, pud, addr);
 8168again:
 8169	pmd = pmdp_get_lockless(pmdp);
 8170	if (!pmd_present(pmd))
 8171		return 0;
 8172
 8173	if (pmd_leaf(pmd))
 8174		return pmd_leaf_size(pmd);
 8175
 8176	ptep = pte_offset_map(&pmd, addr);
 8177	if (!ptep)
 8178		goto again;
 8179
 8180	pte = ptep_get_lockless(ptep);
 8181	if (pte_present(pte))
 8182		size = __pte_leaf_size(pmd, pte);
 8183	pte_unmap(ptep);
 8184#endif /* CONFIG_HAVE_GUP_FAST */
 8185
 8186	return size;
 8187}
 8188
 8189static u64 perf_get_page_size(unsigned long addr)
 8190{
 8191	struct mm_struct *mm;
 8192	unsigned long flags;
 8193	u64 size;
 8194
 8195	if (!addr)
 8196		return 0;
 8197
 8198	/*
 8199	 * Software page-table walkers must disable IRQs,
 8200	 * which prevents any tear down of the page tables.
 8201	 */
 8202	local_irq_save(flags);
 8203
 8204	mm = current->mm;
 8205	if (!mm) {
 8206		/*
 8207		 * For kernel threads and the like, use init_mm so that
 8208		 * we can find kernel memory.
 8209		 */
 8210		mm = &init_mm;
 8211	}
 8212
 8213	size = perf_get_pgtable_size(mm, addr);
 8214
 8215	local_irq_restore(flags);
 8216
 8217	return size;
 8218}
 8219
 8220static struct perf_callchain_entry __empty_callchain = { .nr = 0, };
 8221
 8222static struct unwind_work perf_unwind_work;
 8223
 8224struct perf_callchain_entry *
 8225perf_callchain(struct perf_event *event, struct pt_regs *regs)
 8226{
 8227	bool kernel = !event->attr.exclude_callchain_kernel;
 8228	bool user   = !event->attr.exclude_callchain_user &&
 8229		is_user_task(current);
 8230	/* Disallow cross-task user callchains. */
 8231	bool crosstask = event->ctx->task && event->ctx->task != current;
 8232	bool defer_user = IS_ENABLED(CONFIG_UNWIND_USER) && user &&
 8233			  event->attr.defer_callchain;
 8234	const u32 max_stack = event->attr.sample_max_stack;
 8235	struct perf_callchain_entry *callchain;
 8236	u64 defer_cookie;
 8237
 8238	if (!current->mm)
 8239		user = false;
 8240
 8241	if (!kernel && !user)
 8242		return &__empty_callchain;
 8243
 8244	if (!(user && defer_user && !crosstask &&
 8245	      unwind_deferred_request(&perf_unwind_work, &defer_cookie) >= 0))
 8246		defer_cookie = 0;
 8247
 8248	callchain = get_perf_callchain(regs, kernel, user, max_stack,
 8249				       crosstask, true, defer_cookie);
 8250
 8251	return callchain ?: &__empty_callchain;
 8252}
 8253
 8254static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d)
 8255{
 8256	return d * !!(flags & s);
 8257}
 8258
 8259void perf_prepare_sample(struct perf_sample_data *data,
 8260			 struct perf_event *event,
 8261			 struct pt_regs *regs)
 8262{
 8263	u64 sample_type = event->attr.sample_type;
 8264	u64 filtered_sample_type;
 8265
 8266	/*
 8267	 * Add the sample flags that are dependent to others.  And clear the
 8268	 * sample flags that have already been done by the PMU driver.
 8269	 */
 8270	filtered_sample_type = sample_type;
 8271	filtered_sample_type |= __cond_set(sample_type, PERF_SAMPLE_CODE_PAGE_SIZE,
 8272					   PERF_SAMPLE_IP);
 8273	filtered_sample_type |= __cond_set(sample_type, PERF_SAMPLE_DATA_PAGE_SIZE |
 8274					   PERF_SAMPLE_PHYS_ADDR, PERF_SAMPLE_ADDR);
 8275	filtered_sample_type |= __cond_set(sample_type, PERF_SAMPLE_STACK_USER,
 8276					   PERF_SAMPLE_REGS_USER);
 8277	filtered_sample_type &= ~data->sample_flags;
 8278
 8279	if (filtered_sample_type == 0) {
 8280		/* Make sure it has the correct data->type for output */
 8281		data->type = event->attr.sample_type;
 8282		return;
 8283	}
 8284
 8285	__perf_event_header__init_id(data, event, filtered_sample_type);
 8286
 8287	if (filtered_sample_type & PERF_SAMPLE_IP) {
 8288		data->ip = perf_instruction_pointer(event, regs);
 8289		data->sample_flags |= PERF_SAMPLE_IP;
 8290	}
 8291
 8292	if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN)
 8293		perf_sample_save_callchain(data, event, regs);
 8294
 8295	if (filtered_sample_type & PERF_SAMPLE_RAW) {
 8296		data->raw = NULL;
 8297		data->dyn_size += sizeof(u64);
 8298		data->sample_flags |= PERF_SAMPLE_RAW;
 8299	}
 8300
 8301	if (filtered_sample_type & PERF_SAMPLE_BRANCH_STACK) {
 8302		data->br_stack = NULL;
 8303		data->dyn_size += sizeof(u64);
 8304		data->sample_flags |= PERF_SAMPLE_BRANCH_STACK;
 8305	}
 8306
 8307	if (filtered_sample_type & PERF_SAMPLE_REGS_USER)
 8308		perf_sample_regs_user(&data->regs_user, regs);
 8309
 8310	/*
 8311	 * It cannot use the filtered_sample_type here as REGS_USER can be set
 8312	 * by STACK_USER (using __cond_set() above) and we don't want to update
 8313	 * the dyn_size if it's not requested by users.
 8314	 */
 8315	if ((sample_type & ~data->sample_flags) & PERF_SAMPLE_REGS_USER) {
 8316		/* regs dump ABI info */
 8317		int size = sizeof(u64);
 8318
 8319		if (data->regs_user.regs) {
 8320			u64 mask = event->attr.sample_regs_user;
 8321			size += hweight64(mask) * sizeof(u64);
 8322		}
 8323
 8324		data->dyn_size += size;
 8325		data->sample_flags |= PERF_SAMPLE_REGS_USER;
 8326	}
 8327
 8328	if (filtered_sample_type & PERF_SAMPLE_STACK_USER) {
 8329		/*
 8330		 * Either we need PERF_SAMPLE_STACK_USER bit to be always
 8331		 * processed as the last one or have additional check added
 8332		 * in case new sample type is added, because we could eat
 8333		 * up the rest of the sample size.
 8334		 */
 8335		u16 stack_size = event->attr.sample_stack_user;
 8336		u16 header_size = perf_sample_data_size(data, event);
 8337		u16 size = sizeof(u64);
 8338
 8339		stack_size = perf_sample_ustack_size(stack_size, header_size,
 8340						     data->regs_user.regs);
 8341
 8342		/*
 8343		 * If there is something to dump, add space for the dump
 8344		 * itself and for the field that tells the dynamic size,
 8345		 * which is how many have been actually dumped.
 8346		 */
 8347		if (stack_size)
 8348			size += sizeof(u64) + stack_size;
 8349
 8350		data->stack_user_size = stack_size;
 8351		data->dyn_size += size;
 8352		data->sample_flags |= PERF_SAMPLE_STACK_USER;
 8353	}
 8354
 8355	if (filtered_sample_type & PERF_SAMPLE_WEIGHT_TYPE) {
 8356		data->weight.full = 0;
 8357		data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE;
 8358	}
 8359
 8360	if (filtered_sample_type & PERF_SAMPLE_DATA_SRC) {
 8361		data->data_src.val = PERF_MEM_NA;
 8362		data->sample_flags |= PERF_SAMPLE_DATA_SRC;
 8363	}
 8364
 8365	if (filtered_sample_type & PERF_SAMPLE_TRANSACTION) {
 8366		data->txn = 0;
 8367		data->sample_flags |= PERF_SAMPLE_TRANSACTION;
 8368	}
 8369
 8370	if (filtered_sample_type & PERF_SAMPLE_ADDR) {
 8371		data->addr = 0;
 8372		data->sample_flags |= PERF_SAMPLE_ADDR;
 8373	}
 8374
 8375	if (filtered_sample_type & PERF_SAMPLE_REGS_INTR) {
 8376		/* regs dump ABI info */
 8377		int size = sizeof(u64);
 8378
 8379		perf_sample_regs_intr(&data->regs_intr, regs);
 8380
 8381		if (data->regs_intr.regs) {
 8382			u64 mask = event->attr.sample_regs_intr;
 8383
 8384			size += hweight64(mask) * sizeof(u64);
 8385		}
 8386
 8387		data->dyn_size += size;
 8388		data->sample_flags |= PERF_SAMPLE_REGS_INTR;
 8389	}
 8390
 8391	if (filtered_sample_type & PERF_SAMPLE_PHYS_ADDR) {
 8392		data->phys_addr = perf_virt_to_phys(data->addr);
 8393		data->sample_flags |= PERF_SAMPLE_PHYS_ADDR;
 8394	}
 8395
 8396#ifdef CONFIG_CGROUP_PERF
 8397	if (filtered_sample_type & PERF_SAMPLE_CGROUP) {
 8398		struct cgroup *cgrp;
 8399
 8400		/* protected by RCU */
 8401		cgrp = task_css_check(current, perf_event_cgrp_id, 1)->cgroup;
 8402		data->cgroup = cgroup_id(cgrp);
 8403		data->sample_flags |= PERF_SAMPLE_CGROUP;
 8404	}
 8405#endif
 8406
 8407	/*
 8408	 * PERF_DATA_PAGE_SIZE requires PERF_SAMPLE_ADDR. If the user doesn't
 8409	 * require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr,
 8410	 * but the value will not dump to the userspace.
 8411	 */
 8412	if (filtered_sample_type & PERF_SAMPLE_DATA_PAGE_SIZE) {
 8413		data->data_page_size = perf_get_page_size(data->addr);
 8414		data->sample_flags |= PERF_SAMPLE_DATA_PAGE_SIZE;
 8415	}
 8416
 8417	if (filtered_sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) {
 8418		data->code_page_size = perf_get_page_size(data->ip);
 8419		data->sample_flags |= PERF_SAMPLE_CODE_PAGE_SIZE;
 8420	}
 8421
 8422	if (filtered_sample_type & PERF_SAMPLE_AUX) {
 8423		u64 size;
 8424		u16 header_size = perf_sample_data_size(data, event);
 8425
 8426		header_size += sizeof(u64); /* size */
 8427
 8428		/*
 8429		 * Given the 16bit nature of header::size, an AUX sample can
 8430		 * easily overflow it, what with all the preceding sample bits.
 8431		 * Make sure this doesn't happen by using up to U16_MAX bytes
 8432		 * per sample in total (rounded down to 8 byte boundary).
 8433		 */
 8434		size = min_t(size_t, U16_MAX - header_size,
 8435			     event->attr.aux_sample_size);
 8436		size = rounddown(size, 8);
 8437		size = perf_prepare_sample_aux(event, data, size);
 8438
 8439		WARN_ON_ONCE(size + header_size > U16_MAX);
 8440		data->dyn_size += size + sizeof(u64); /* size above */
 8441		data->sample_flags |= PERF_SAMPLE_AUX;
 8442	}
 8443}
 8444
 8445void perf_prepare_header(struct perf_event_header *header,
 8446			 struct perf_sample_data *data,
 8447			 struct perf_event *event,
 8448			 struct pt_regs *regs)
 8449{
 8450	header->type = PERF_RECORD_SAMPLE;
 8451	header->size = perf_sample_data_size(data, event);
 8452	header->misc = perf_misc_flags(event, regs);
 8453
 8454	/*
 8455	 * If you're adding more sample types here, you likely need to do
 8456	 * something about the overflowing header::size, like repurpose the
 8457	 * lowest 3 bits of size, which should be always zero at the moment.
 8458	 * This raises a more important question, do we really need 512k sized
 8459	 * samples and why, so good argumentation is in order for whatever you
 8460	 * do here next.
 8461	 */
 8462	WARN_ON_ONCE(header->size & 7);
 8463}
 8464
 8465static void __perf_event_aux_pause(struct perf_event *event, bool pause)
 8466{
 8467	if (pause) {
 8468		if (!event->hw.aux_paused) {
 8469			event->hw.aux_paused = 1;
 8470			event->pmu->stop(event, PERF_EF_PAUSE);
 8471		}
 8472	} else {
 8473		if (event->hw.aux_paused) {
 8474			event->hw.aux_paused = 0;
 8475			event->pmu->start(event, PERF_EF_RESUME);
 8476		}
 8477	}
 8478}
 8479
 8480static void perf_event_aux_pause(struct perf_event *event, bool pause)
 8481{
 8482	struct perf_buffer *rb;
 8483
 8484	if (WARN_ON_ONCE(!event))
 8485		return;
 8486
 8487	rb = ring_buffer_get(event);
 8488	if (!rb)
 8489		return;
 8490
 8491	scoped_guard (irqsave) {
 8492		/*
 8493		 * Guard against self-recursion here. Another event could trip
 8494		 * this same from NMI context.
 8495		 */
 8496		if (READ_ONCE(rb->aux_in_pause_resume))
 8497			break;
 8498
 8499		WRITE_ONCE(rb->aux_in_pause_resume, 1);
 8500		barrier();
 8501		__perf_event_aux_pause(event, pause);
 8502		barrier();
 8503		WRITE_ONCE(rb->aux_in_pause_resume, 0);
 8504	}
 8505	ring_buffer_put(rb);
 8506}
 8507
 8508static __always_inline int
 8509__perf_event_output(struct perf_event *event,
 8510		    struct perf_sample_data *data,
 8511		    struct pt_regs *regs,
 8512		    int (*output_begin)(struct perf_output_handle *,
 8513					struct perf_sample_data *,
 8514					struct perf_event *,
 8515					unsigned int))
 8516{
 8517	struct perf_output_handle handle;
 8518	struct perf_event_header header;
 8519	int err;
 8520
 8521	/* protect the callchain buffers */
 8522	rcu_read_lock();
 8523
 8524	perf_prepare_sample(data, event, regs);
 8525	perf_prepare_header(&header, data, event, regs);
 8526
 8527	err = output_begin(&handle, data, event, header.size);
 8528	if (err)
 8529		goto exit;
 8530
 8531	perf_output_sample(&handle, &header, data, event);
 8532
 8533	perf_output_end(&handle);
 8534
 8535exit:
 8536	rcu_read_unlock();
 8537	return err;
 8538}
 8539
 8540void
 8541perf_event_output_forward(struct perf_event *event,
 8542			 struct perf_sample_data *data,
 8543			 struct pt_regs *regs)
 8544{
 8545	__perf_event_output(event, data, regs, perf_output_begin_forward);
 8546}
 8547
 8548void
 8549perf_event_output_backward(struct perf_event *event,
 8550			   struct perf_sample_data *data,
 8551			   struct pt_regs *regs)
 8552{
 8553	__perf_event_output(event, data, regs, perf_output_begin_backward);
 8554}
 8555
 8556int
 8557perf_event_output(struct perf_event *event,
 8558		  struct perf_sample_data *data,
 8559		  struct pt_regs *regs)
 8560{
 8561	return __perf_event_output(event, data, regs, perf_output_begin);
 8562}
 8563
 8564/*
 8565 * read event_id
 8566 */
 8567
 8568struct perf_read_event {
 8569	struct perf_event_header	header;
 8570
 8571	u32				pid;
 8572	u32				tid;
 8573};
 8574
 8575static void
 8576perf_event_read_event(struct perf_event *event,
 8577			struct task_struct *task)
 8578{
 8579	struct perf_output_handle handle;
 8580	struct perf_sample_data sample;
 8581	struct perf_read_event read_event = {
 8582		.header = {
 8583			.type = PERF_RECORD_READ,
 8584			.misc = 0,
 8585			.size = sizeof(read_event) + event->read_size,
 8586		},
 8587		.pid = perf_event_pid(event, task),
 8588		.tid = perf_event_tid(event, task),
 8589	};
 8590	int ret;
 8591
 8592	perf_event_header__init_id(&read_event.header, &sample, event);
 8593	ret = perf_output_begin(&handle, &sample, event, read_event.header.size);
 8594	if (ret)
 8595		return;
 8596
 8597	perf_output_put(&handle, read_event);
 8598	perf_output_read(&handle, event);
 8599	perf_event__output_id_sample(event, &handle, &sample);
 8600
 8601	perf_output_end(&handle);
 8602}
 8603
 8604typedef void (perf_iterate_f)(struct perf_event *event, void *data);
 8605
 8606static void
 8607perf_iterate_ctx(struct perf_event_context *ctx,
 8608		   perf_iterate_f output,
 8609		   void *data, bool all)
 8610{
 8611	struct perf_event *event;
 8612
 8613	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
 8614		if (!all) {
 8615			if (event->state < PERF_EVENT_STATE_INACTIVE)
 8616				continue;
 8617			if (!event_filter_match(event))
 8618				continue;
 8619		}
 8620
 8621		output(event, data);
 8622	}
 8623}
 8624
 8625static void perf_iterate_sb_cpu(perf_iterate_f output, void *data)
 8626{
 8627	struct pmu_event_list *pel = this_cpu_ptr(&pmu_sb_events);
 8628	struct perf_event *event;
 8629
 8630	list_for_each_entry_rcu(event, &pel->list, sb_list) {
 8631		/*
 8632		 * Skip events that are not fully formed yet; ensure that
 8633		 * if we observe event->ctx, both event and ctx will be
 8634		 * complete enough. See perf_install_in_context().
 8635		 */
 8636		if (!smp_load_acquire(&event->ctx))
 8637			continue;
 8638
 8639		if (event->state < PERF_EVENT_STATE_INACTIVE)
 8640			continue;
 8641		if (!event_filter_match(event))
 8642			continue;
 8643		output(event, data);
 8644	}
 8645}
 8646
 8647/*
 8648 * Iterate all events that need to receive side-band events.
 8649 *
 8650 * For new callers; ensure that account_pmu_sb_event() includes
 8651 * your event, otherwise it might not get delivered.
 8652 */
 8653static void
 8654perf_iterate_sb(perf_iterate_f output, void *data,
 8655	       struct perf_event_context *task_ctx)
 8656{
 8657	struct perf_event_context *ctx;
 8658
 8659	rcu_read_lock();
 8660	preempt_disable();
 8661
 8662	/*
 8663	 * If we have task_ctx != NULL we only notify the task context itself.
 8664	 * The task_ctx is set only for EXIT events before releasing task
 8665	 * context.
 8666	 */
 8667	if (task_ctx) {
 8668		perf_iterate_ctx(task_ctx, output, data, false);
 8669		goto done;
 8670	}
 8671
 8672	perf_iterate_sb_cpu(output, data);
 8673
 8674	ctx = rcu_dereference(current->perf_event_ctxp);
 8675	if (ctx)
 8676		perf_iterate_ctx(ctx, output, data, false);
 8677done:
 8678	preempt_enable();
 8679	rcu_read_unlock();
 8680}
 8681
 8682/*
 8683 * Clear all file-based filters at exec, they'll have to be
 8684 * re-instated when/if these objects are mmapped again.
 8685 */
 8686static void perf_event_addr_filters_exec(struct perf_event *event, void *data)
 8687{
 8688	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
 8689	struct perf_addr_filter *filter;
 8690	unsigned int restart = 0, count = 0;
 8691	unsigned long flags;
 8692
 8693	if (!has_addr_filter(event))
 8694		return;
 8695
 8696	raw_spin_lock_irqsave(&ifh->lock, flags);
 8697	list_for_each_entry(filter, &ifh->list, entry) {
 8698		if (filter->path.dentry) {
 8699			event->addr_filter_ranges[count].start = 0;
 8700			event->addr_filter_ranges[count].size = 0;
 8701			restart++;
 8702		}
 8703
 8704		count++;
 8705	}
 8706
 8707	if (restart)
 8708		event->addr_filters_gen++;
 8709	raw_spin_unlock_irqrestore(&ifh->lock, flags);
 8710
 8711	if (restart)
 8712		perf_event_stop(event, 1);
 8713}
 8714
 8715void perf_event_exec(void)
 8716{
 8717	struct perf_event_context *ctx;
 8718
 8719	ctx = perf_pin_task_context(current);
 8720	if (!ctx)
 8721		return;
 8722
 8723	perf_event_enable_on_exec(ctx);
 8724	perf_event_remove_on_exec(ctx);
 8725	scoped_guard(rcu)
 8726		perf_iterate_ctx(ctx, perf_event_addr_filters_exec, NULL, true);
 8727
 8728	perf_unpin_context(ctx);
 8729	put_ctx(ctx);
 8730}
 8731
 8732struct remote_output {
 8733	struct perf_buffer	*rb;
 8734	int			err;
 8735};
 8736
 8737static void __perf_event_output_stop(struct perf_event *event, void *data)
 8738{
 8739	struct perf_event *parent = event->parent;
 8740	struct remote_output *ro = data;
 8741	struct perf_buffer *rb = ro->rb;
 8742	struct stop_event_data sd = {
 8743		.event	= event,
 8744	};
 8745
 8746	if (!has_aux(event))
 8747		return;
 8748
 8749	if (!parent)
 8750		parent = event;
 8751
 8752	/*
 8753	 * In case of inheritance, it will be the parent that links to the
 8754	 * ring-buffer, but it will be the child that's actually using it.
 8755	 *
 8756	 * We are using event::rb to determine if the event should be stopped,
 8757	 * however this may race with ring_buffer_attach() (through set_output),
 8758	 * which will make us skip the event that actually needs to be stopped.
 8759	 * So ring_buffer_attach() has to stop an aux event before re-assigning
 8760	 * its rb pointer.
 8761	 */
 8762	if (rcu_dereference(parent->rb) == rb)
 8763		ro->err = __perf_event_stop(&sd);
 8764}
 8765
 8766static int __perf_pmu_output_stop(void *info)
 8767{
 8768	struct perf_event *event = info;
 8769	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
 8770	struct remote_output ro = {
 8771		.rb	= event->rb,
 8772	};
 8773
 8774	rcu_read_lock();
 8775	perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false);
 8776	if (cpuctx->task_ctx)
 8777		perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop,
 8778				   &ro, false);
 8779	rcu_read_unlock();
 8780
 8781	return ro.err;
 8782}
 8783
 8784static void perf_pmu_output_stop(struct perf_event *event)
 8785{
 8786	struct perf_event *iter;
 8787	int err, cpu;
 8788
 8789restart:
 8790	rcu_read_lock();
 8791	list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) {
 8792		/*
 8793		 * For per-CPU events, we need to make sure that neither they
 8794		 * nor their children are running; for cpu==-1 events it's
 8795		 * sufficient to stop the event itself if it's active, since
 8796		 * it can't have children.
 8797		 */
 8798		cpu = iter->cpu;
 8799		if (cpu == -1)
 8800			cpu = READ_ONCE(iter->oncpu);
 8801
 8802		if (cpu == -1)
 8803			continue;
 8804
 8805		err = cpu_function_call(cpu, __perf_pmu_output_stop, event);
 8806		if (err == -EAGAIN) {
 8807			rcu_read_unlock();
 8808			goto restart;
 8809		}
 8810	}
 8811	rcu_read_unlock();
 8812}
 8813
 8814/*
 8815 * task tracking -- fork/exit
 8816 *
 8817 * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task
 8818 */
 8819
 8820struct perf_task_event {
 8821	struct task_struct		*task;
 8822	struct perf_event_context	*task_ctx;
 8823
 8824	struct {
 8825		struct perf_event_header	header;
 8826
 8827		u32				pid;
 8828		u32				ppid;
 8829		u32				tid;
 8830		u32				ptid;
 8831		u64				time;
 8832	} event_id;
 8833};
 8834
 8835static int perf_event_task_match(struct perf_event *event)
 8836{
 8837	return event->attr.comm  || event->attr.mmap ||
 8838	       event->attr.mmap2 || event->attr.mmap_data ||
 8839	       event->attr.task;
 8840}
 8841
 8842static void perf_event_task_output(struct perf_event *event,
 8843				   void *data)
 8844{
 8845	struct perf_task_event *task_event = data;
 8846	struct perf_output_handle handle;
 8847	struct perf_sample_data	sample;
 8848	struct task_struct *task = task_event->task;
 8849	int ret, size = task_event->event_id.header.size;
 8850
 8851	if (!perf_event_task_match(event))
 8852		return;
 8853
 8854	perf_event_header__init_id(&task_event->event_id.header, &sample, event);
 8855
 8856	ret = perf_output_begin(&handle, &sample, event,
 8857				task_event->event_id.header.size);
 8858	if (ret)
 8859		goto out;
 8860
 8861	task_event->event_id.pid = perf_event_pid(event, task);
 8862	task_event->event_id.tid = perf_event_tid(event, task);
 8863
 8864	if (task_event->event_id.header.type == PERF_RECORD_EXIT) {
 8865		task_event->event_id.ppid = perf_event_pid(event,
 8866							task->real_parent);
 8867		task_event->event_id.ptid = perf_event_pid(event,
 8868							task->real_parent);
 8869	} else {  /* PERF_RECORD_FORK */
 8870		task_event->event_id.ppid = perf_event_pid(event, current);
 8871		task_event->event_id.ptid = perf_event_tid(event, current);
 8872	}
 8873
 8874	task_event->event_id.time = perf_event_clock(event);
 8875
 8876	perf_output_put(&handle, task_event->event_id);
 8877
 8878	perf_event__output_id_sample(event, &handle, &sample);
 8879
 8880	perf_output_end(&handle);
 8881out:
 8882	task_event->event_id.header.size = size;
 8883}
 8884
 8885static void perf_event_task(struct task_struct *task,
 8886			      struct perf_event_context *task_ctx,
 8887			      int new)
 8888{
 8889	struct perf_task_event task_event;
 8890
 8891	if (!atomic_read(&nr_comm_events) &&
 8892	    !atomic_read(&nr_mmap_events) &&
 8893	    !atomic_read(&nr_task_events))
 8894		return;
 8895
 8896	task_event = (struct perf_task_event){
 8897		.task	  = task,
 8898		.task_ctx = task_ctx,
 8899		.event_id    = {
 8900			.header = {
 8901				.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
 8902				.misc = 0,
 8903				.size = sizeof(task_event.event_id),
 8904			},
 8905			/* .pid  */
 8906			/* .ppid */
 8907			/* .tid  */
 8908			/* .ptid */
 8909			/* .time */
 8910		},
 8911	};
 8912
 8913	perf_iterate_sb(perf_event_task_output,
 8914		       &task_event,
 8915		       task_ctx);
 8916}
 8917
 8918/*
 8919 * Allocate data for a new task when profiling system-wide
 8920 * events which require PMU specific data
 8921 */
 8922static void
 8923perf_event_alloc_task_data(struct task_struct *child,
 8924			   struct task_struct *parent)
 8925{
 8926	struct kmem_cache *ctx_cache = NULL;
 8927	struct perf_ctx_data *cd;
 8928
 8929	if (!refcount_read(&global_ctx_data_ref))
 8930		return;
 8931
 8932	scoped_guard (rcu) {
 8933		cd = rcu_dereference(parent->perf_ctx_data);
 8934		if (cd)
 8935			ctx_cache = cd->ctx_cache;
 8936	}
 8937
 8938	if (!ctx_cache)
 8939		return;
 8940
 8941	guard(percpu_read)(&global_ctx_data_rwsem);
 8942	scoped_guard (rcu) {
 8943		cd = rcu_dereference(child->perf_ctx_data);
 8944		if (!cd) {
 8945			/*
 8946			 * A system-wide event may be unaccount,
 8947			 * when attaching the perf_ctx_data.
 8948			 */
 8949			if (!refcount_read(&global_ctx_data_ref))
 8950				return;
 8951			goto attach;
 8952		}
 8953
 8954		if (!cd->global) {
 8955			cd->global = 1;
 8956			refcount_inc(&cd->refcount);
 8957		}
 8958	}
 8959
 8960	return;
 8961attach:
 8962	attach_task_ctx_data(child, ctx_cache, true);
 8963}
 8964
 8965void perf_event_fork(struct task_struct *task)
 8966{
 8967	perf_event_task(task, NULL, 1);
 8968	perf_event_namespaces(task);
 8969	perf_event_alloc_task_data(task, current);
 8970}
 8971
 8972/*
 8973 * comm tracking
 8974 */
 8975
 8976struct perf_comm_event {
 8977	struct task_struct	*task;
 8978	char			*comm;
 8979	int			comm_size;
 8980
 8981	struct {
 8982		struct perf_event_header	header;
 8983
 8984		u32				pid;
 8985		u32				tid;
 8986	} event_id;
 8987};
 8988
 8989static int perf_event_comm_match(struct perf_event *event)
 8990{
 8991	return event->attr.comm;
 8992}
 8993
 8994static void perf_event_comm_output(struct perf_event *event,
 8995				   void *data)
 8996{
 8997	struct perf_comm_event *comm_event = data;
 8998	struct perf_output_handle handle;
 8999	struct perf_sample_data sample;
 9000	int size = comm_event->event_id.header.size;
 9001	int ret;
 9002
 9003	if (!perf_event_comm_match(event))
 9004		return;
 9005
 9006	perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
 9007	ret = perf_output_begin(&handle, &sample, event,
 9008				comm_event->event_id.header.size);
 9009
 9010	if (ret)
 9011		goto out;
 9012
 9013	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
 9014	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
 9015
 9016	perf_output_put(&handle, comm_event->event_id);
 9017	__output_copy(&handle, comm_event->comm,
 9018				   comm_event->comm_size);
 9019
 9020	perf_event__output_id_sample(event, &handle, &sample);
 9021
 9022	perf_output_end(&handle);
 9023out:
 9024	comm_event->event_id.header.size = size;
 9025}
 9026
 9027static void perf_event_comm_event(struct perf_comm_event *comm_event)
 9028{
 9029	char comm[TASK_COMM_LEN];
 9030	unsigned int size;
 9031
 9032	memset(comm, 0, sizeof(comm));
 9033	strscpy(comm, comm_event->task->comm);
 9034	size = ALIGN(strlen(comm)+1, sizeof(u64));
 9035
 9036	comm_event->comm = comm;
 9037	comm_event->comm_size = size;
 9038
 9039	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
 9040
 9041	perf_iterate_sb(perf_event_comm_output,
 9042		       comm_event,
 9043		       NULL);
 9044}
 9045
 9046void perf_event_comm(struct task_struct *task, bool exec)
 9047{
 9048	struct perf_comm_event comm_event;
 9049
 9050	if (!atomic_read(&nr_comm_events))
 9051		return;
 9052
 9053	comm_event = (struct perf_comm_event){
 9054		.task	= task,
 9055		/* .comm      */
 9056		/* .comm_size */
 9057		.event_id  = {
 9058			.header = {
 9059				.type = PERF_RECORD_COMM,
 9060				.misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0,
 9061				/* .size */
 9062			},
 9063			/* .pid */
 9064			/* .tid */
 9065		},
 9066	};
 9067
 9068	perf_event_comm_event(&comm_event);
 9069}
 9070
 9071/*
 9072 * namespaces tracking
 9073 */
 9074
 9075struct perf_namespaces_event {
 9076	struct task_struct		*task;
 9077
 9078	struct {
 9079		struct perf_event_header	header;
 9080
 9081		u32				pid;
 9082		u32				tid;
 9083		u64				nr_namespaces;
 9084		struct perf_ns_link_info	link_info[NR_NAMESPACES];
 9085	} event_id;
 9086};
 9087
 9088static int perf_event_namespaces_match(struct perf_event *event)
 9089{
 9090	return event->attr.namespaces;
 9091}
 9092
 9093static void perf_event_namespaces_output(struct perf_event *event,
 9094					 void *data)
 9095{
 9096	struct perf_namespaces_event *namespaces_event = data;
 9097	struct perf_output_handle handle;
 9098	struct perf_sample_data sample;
 9099	u16 header_size = namespaces_event->event_id.header.size;
 9100	int ret;
 9101
 9102	if (!perf_event_namespaces_match(event))
 9103		return;
 9104
 9105	perf_event_header__init_id(&namespaces_event->event_id.header,
 9106				   &sample, event);
 9107	ret = perf_output_begin(&handle, &sample, event,
 9108				namespaces_event->event_id.header.size);
 9109	if (ret)
 9110		goto out;
 9111
 9112	namespaces_event->event_id.pid = perf_event_pid(event,
 9113							namespaces_event->task);
 9114	namespaces_event->event_id.tid = perf_event_tid(event,
 9115							namespaces_event->task);
 9116
 9117	perf_output_put(&handle, namespaces_event->event_id);
 9118
 9119	perf_event__output_id_sample(event, &handle, &sample);
 9120
 9121	perf_output_end(&handle);
 9122out:
 9123	namespaces_event->event_id.header.size = header_size;
 9124}
 9125
 9126static void perf_fill_ns_link_info(struct perf_ns_link_info *ns_link_info,
 9127				   struct task_struct *task,
 9128				   const struct proc_ns_operations *ns_ops)
 9129{
 9130	struct path ns_path;
 9131	struct inode *ns_inode;
 9132	int error;
 9133
 9134	error = ns_get_path(&ns_path, task, ns_ops);
 9135	if (!error) {
 9136		ns_inode = ns_path.dentry->d_inode;
 9137		ns_link_info->dev = new_encode_dev(ns_inode->i_sb->s_dev);
 9138		ns_link_info->ino = ns_inode->i_ino;
 9139		path_put(&ns_path);
 9140	}
 9141}
 9142
 9143void perf_event_namespaces(struct task_struct *task)
 9144{
 9145	struct perf_namespaces_event namespaces_event;
 9146	struct perf_ns_link_info *ns_link_info;
 9147
 9148	if (!atomic_read(&nr_namespaces_events))
 9149		return;
 9150
 9151	namespaces_event = (struct perf_namespaces_event){
 9152		.task	= task,
 9153		.event_id  = {
 9154			.header = {
 9155				.type = PERF_RECORD_NAMESPACES,
 9156				.misc = 0,
 9157				.size = sizeof(namespaces_event.event_id),
 9158			},
 9159			/* .pid */
 9160			/* .tid */
 9161			.nr_namespaces = NR_NAMESPACES,
 9162			/* .link_info[NR_NAMESPACES] */
 9163		},
 9164	};
 9165
 9166	ns_link_info = namespaces_event.event_id.link_info;
 9167
 9168	perf_fill_ns_link_info(&ns_link_info[MNT_NS_INDEX],
 9169			       task, &mntns_operations);
 9170
 9171#ifdef CONFIG_USER_NS
 9172	perf_fill_ns_link_info(&ns_link_info[USER_NS_INDEX],
 9173			       task, &userns_operations);
 9174#endif
 9175#ifdef CONFIG_NET_NS
 9176	perf_fill_ns_link_info(&ns_link_info[NET_NS_INDEX],
 9177			       task, &netns_operations);
 9178#endif
 9179#ifdef CONFIG_UTS_NS
 9180	perf_fill_ns_link_info(&ns_link_info[UTS_NS_INDEX],
 9181			       task, &utsns_operations);
 9182#endif
 9183#ifdef CONFIG_IPC_NS
 9184	perf_fill_ns_link_info(&ns_link_info[IPC_NS_INDEX],
 9185			       task, &ipcns_operations);
 9186#endif
 9187#ifdef CONFIG_PID_NS
 9188	perf_fill_ns_link_info(&ns_link_info[PID_NS_INDEX],
 9189			       task, &pidns_operations);
 9190#endif
 9191#ifdef CONFIG_CGROUPS
 9192	perf_fill_ns_link_info(&ns_link_info[CGROUP_NS_INDEX],
 9193			       task, &cgroupns_operations);
 9194#endif
 9195
 9196	perf_iterate_sb(perf_event_namespaces_output,
 9197			&namespaces_event,
 9198			NULL);
 9199}
 9200
 9201/*
 9202 * cgroup tracking
 9203 */
 9204#ifdef CONFIG_CGROUP_PERF
 9205
 9206struct perf_cgroup_event {
 9207	char				*path;
 9208	int				path_size;
 9209	struct {
 9210		struct perf_event_header	header;
 9211		u64				id;
 9212		char				path[];
 9213	} event_id;
 9214};
 9215
 9216static int perf_event_cgroup_match(struct perf_event *event)
 9217{
 9218	return event->attr.cgroup;
 9219}
 9220
 9221static void perf_event_cgroup_output(struct perf_event *event, void *data)
 9222{
 9223	struct perf_cgroup_event *cgroup_event = data;
 9224	struct perf_output_handle handle;
 9225	struct perf_sample_data sample;
 9226	u16 header_size = cgroup_event->event_id.header.size;
 9227	int ret;
 9228
 9229	if (!perf_event_cgroup_match(event))
 9230		return;
 9231
 9232	perf_event_header__init_id(&cgroup_event->event_id.header,
 9233				   &sample, event);
 9234	ret = perf_output_begin(&handle, &sample, event,
 9235				cgroup_event->event_id.header.size);
 9236	if (ret)
 9237		goto out;
 9238
 9239	perf_output_put(&handle, cgroup_event->event_id);
 9240	__output_copy(&handle, cgroup_event->path, cgroup_event->path_size);
 9241
 9242	perf_event__output_id_sample(event, &handle, &sample);
 9243
 9244	perf_output_end(&handle);
 9245out:
 9246	cgroup_event->event_id.header.size = header_size;
 9247}
 9248
 9249static void perf_event_cgroup(struct cgroup *cgrp)
 9250{
 9251	struct perf_cgroup_event cgroup_event;
 9252	char path_enomem[16] = "//enomem";
 9253	char *pathname;
 9254	size_t size;
 9255
 9256	if (!atomic_read(&nr_cgroup_events))
 9257		return;
 9258
 9259	cgroup_event = (struct perf_cgroup_event){
 9260		.event_id  = {
 9261			.header = {
 9262				.type = PERF_RECORD_CGROUP,
 9263				.misc = 0,
 9264				.size = sizeof(cgroup_event.event_id),
 9265			},
 9266			.id = cgroup_id(cgrp),
 9267		},
 9268	};
 9269
 9270	pathname = kmalloc(PATH_MAX, GFP_KERNEL);
 9271	if (pathname == NULL) {
 9272		cgroup_event.path = path_enomem;
 9273	} else {
 9274		/* just to be sure to have enough space for alignment */
 9275		cgroup_path(cgrp, pathname, PATH_MAX - sizeof(u64));
 9276		cgroup_event.path = pathname;
 9277	}
 9278
 9279	/*
 9280	 * Since our buffer works in 8 byte units we need to align our string
 9281	 * size to a multiple of 8. However, we must guarantee the tail end is
 9282	 * zero'd out to avoid leaking random bits to userspace.
 9283	 */
 9284	size = strlen(cgroup_event.path) + 1;
 9285	while (!IS_ALIGNED(size, sizeof(u64)))
 9286		cgroup_event.path[size++] = '\0';
 9287
 9288	cgroup_event.event_id.header.size += size;
 9289	cgroup_event.path_size = size;
 9290
 9291	perf_iterate_sb(perf_event_cgroup_output,
 9292			&cgroup_event,
 9293			NULL);
 9294
 9295	kfree(pathname);
 9296}
 9297
 9298#endif
 9299
 9300/*
 9301 * mmap tracking
 9302 */
 9303
 9304struct perf_mmap_event {
 9305	struct vm_area_struct	*vma;
 9306
 9307	const char		*file_name;
 9308	int			file_size;
 9309	int			maj, min;
 9310	u64			ino;
 9311	u64			ino_generation;
 9312	u32			prot, flags;
 9313	u8			build_id[BUILD_ID_SIZE_MAX];
 9314	u32			build_id_size;
 9315
 9316	struct {
 9317		struct perf_event_header	header;
 9318
 9319		u32				pid;
 9320		u32				tid;
 9321		u64				start;
 9322		u64				len;
 9323		u64				pgoff;
 9324	} event_id;
 9325};
 9326
 9327static int perf_event_mmap_match(struct perf_event *event,
 9328				 void *data)
 9329{
 9330	struct perf_mmap_event *mmap_event = data;
 9331	struct vm_area_struct *vma = mmap_event->vma;
 9332	int executable = vma->vm_flags & VM_EXEC;
 9333
 9334	return (!executable && event->attr.mmap_data) ||
 9335	       (executable && (event->attr.mmap || event->attr.mmap2));
 9336}
 9337
 9338static void perf_event_mmap_output(struct perf_event *event,
 9339				   void *data)
 9340{
 9341	struct perf_mmap_event *mmap_event = data;
 9342	struct perf_output_handle handle;
 9343	struct perf_sample_data sample;
 9344	int size = mmap_event->event_id.header.size;
 9345	u32 type = mmap_event->event_id.header.type;
 9346	bool use_build_id;
 9347	int ret;
 9348
 9349	if (!perf_event_mmap_match(event, data))
 9350		return;
 9351
 9352	if (event->attr.mmap2) {
 9353		mmap_event->event_id.header.type = PERF_RECORD_MMAP2;
 9354		mmap_event->event_id.header.size += sizeof(mmap_event->maj);
 9355		mmap_event->event_id.header.size += sizeof(mmap_event->min);
 9356		mmap_event->event_id.header.size += sizeof(mmap_event->ino);
 9357		mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation);
 9358		mmap_event->event_id.header.size += sizeof(mmap_event->prot);
 9359		mmap_event->event_id.header.size += sizeof(mmap_event->flags);
 9360	}
 9361
 9362	perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
 9363	ret = perf_output_begin(&handle, &sample, event,
 9364				mmap_event->event_id.header.size);
 9365	if (ret)
 9366		goto out;
 9367
 9368	mmap_event->event_id.pid = perf_event_pid(event, current);
 9369	mmap_event->event_id.tid = perf_event_tid(event, current);
 9370
 9371	use_build_id = event->attr.build_id && mmap_event->build_id_size;
 9372
 9373	if (event->attr.mmap2 && use_build_id)
 9374		mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID;
 9375
 9376	perf_output_put(&handle, mmap_event->event_id);
 9377
 9378	if (event->attr.mmap2) {
 9379		if (use_build_id) {
 9380			u8 size[4] = { (u8) mmap_event->build_id_size, 0, 0, 0 };
 9381
 9382			__output_copy(&handle, size, 4);
 9383			__output_copy(&handle, mmap_event->build_id, BUILD_ID_SIZE_MAX);
 9384		} else {
 9385			perf_output_put(&handle, mmap_event->maj);
 9386			perf_output_put(&handle, mmap_event->min);
 9387			perf_output_put(&handle, mmap_event->ino);
 9388			perf_output_put(&handle, mmap_event->ino_generation);
 9389		}
 9390		perf_output_put(&handle, mmap_event->prot);
 9391		perf_output_put(&handle, mmap_event->flags);
 9392	}
 9393
 9394	__output_copy(&handle, mmap_event->file_name,
 9395				   mmap_event->file_size);
 9396
 9397	perf_event__output_id_sample(event, &handle, &sample);
 9398
 9399	perf_output_end(&handle);
 9400out:
 9401	mmap_event->event_id.header.size = size;
 9402	mmap_event->event_id.header.type = type;
 9403}
 9404
 9405static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
 9406{
 9407	struct vm_area_struct *vma = mmap_event->vma;
 9408	struct file *file = vma->vm_file;
 9409	int maj = 0, min = 0;
 9410	u64 ino = 0, gen = 0;
 9411	u32 prot = 0, flags = 0;
 9412	unsigned int size;
 9413	char tmp[16];
 9414	char *buf = NULL;
 9415	char *name = NULL;
 9416
 9417	if (vma->vm_flags & VM_READ)
 9418		prot |= PROT_READ;
 9419	if (vma->vm_flags & VM_WRITE)
 9420		prot |= PROT_WRITE;
 9421	if (vma->vm_flags & VM_EXEC)
 9422		prot |= PROT_EXEC;
 9423
 9424	if (vma->vm_flags & VM_MAYSHARE)
 9425		flags = MAP_SHARED;
 9426	else
 9427		flags = MAP_PRIVATE;
 9428
 9429	if (vma->vm_flags & VM_LOCKED)
 9430		flags |= MAP_LOCKED;
 9431	if (is_vm_hugetlb_page(vma))
 9432		flags |= MAP_HUGETLB;
 9433
 9434	if (file) {
 9435		const struct inode *inode;
 9436		dev_t dev;
 9437
 9438		buf = kmalloc(PATH_MAX, GFP_KERNEL);
 9439		if (!buf) {
 9440			name = "//enomem";
 9441			goto cpy_name;
 9442		}
 9443		/*
 9444		 * d_path() works from the end of the rb backwards, so we
 9445		 * need to add enough zero bytes after the string to handle
 9446		 * the 64bit alignment we do later.
 9447		 */
 9448		name = d_path(file_user_path(file), buf, PATH_MAX - sizeof(u64));
 9449		if (IS_ERR(name)) {
 9450			name = "//toolong";
 9451			goto cpy_name;
 9452		}
 9453		inode = file_user_inode(vma->vm_file);
 9454		dev = inode->i_sb->s_dev;
 9455		ino = inode->i_ino;
 9456		gen = inode->i_generation;
 9457		maj = MAJOR(dev);
 9458		min = MINOR(dev);
 9459
 9460		goto got_name;
 9461	} else {
 9462		if (vma->vm_ops && vma->vm_ops->name)
 9463			name = (char *) vma->vm_ops->name(vma);
 9464		if (!name)
 9465			name = (char *)arch_vma_name(vma);
 9466		if (!name) {
 9467			if (vma_is_initial_heap(vma))
 9468				name = "[heap]";
 9469			else if (vma_is_initial_stack(vma))
 9470				name = "[stack]";
 9471			else
 9472				name = "//anon";
 9473		}
 9474	}
 9475
 9476cpy_name:
 9477	strscpy(tmp, name);
 9478	name = tmp;
 9479got_name:
 9480	/*
 9481	 * Since our buffer works in 8 byte units we need to align our string
 9482	 * size to a multiple of 8. However, we must guarantee the tail end is
 9483	 * zero'd out to avoid leaking random bits to userspace.
 9484	 */
 9485	size = strlen(name)+1;
 9486	while (!IS_ALIGNED(size, sizeof(u64)))
 9487		name[size++] = '\0';
 9488
 9489	mmap_event->file_name = name;
 9490	mmap_event->file_size = size;
 9491	mmap_event->maj = maj;
 9492	mmap_event->min = min;
 9493	mmap_event->ino = ino;
 9494	mmap_event->ino_generation = gen;
 9495	mmap_event->prot = prot;
 9496	mmap_event->flags = flags;
 9497
 9498	if (!(vma->vm_flags & VM_EXEC))
 9499		mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
 9500
 9501	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
 9502
 9503	if (atomic_read(&nr_build_id_events))
 9504		build_id_parse_nofault(vma, mmap_event->build_id, &mmap_event->build_id_size);
 9505
 9506	perf_iterate_sb(perf_event_mmap_output,
 9507		       mmap_event,
 9508		       NULL);
 9509
 9510	kfree(buf);
 9511}
 9512
 9513/*
 9514 * Check whether inode and address range match filter criteria.
 9515 */
 9516static bool perf_addr_filter_match(struct perf_addr_filter *filter,
 9517				     struct file *file, unsigned long offset,
 9518				     unsigned long size)
 9519{
 9520	/* d_inode(NULL) won't be equal to any mapped user-space file */
 9521	if (!filter->path.dentry)
 9522		return false;
 9523
 9524	if (d_inode(filter->path.dentry) != file_user_inode(file))
 9525		return false;
 9526
 9527	if (filter->offset > offset + size)
 9528		return false;
 9529
 9530	if (filter->offset + filter->size < offset)
 9531		return false;
 9532
 9533	return true;
 9534}
 9535
 9536static bool perf_addr_filter_vma_adjust(struct perf_addr_filter *filter,
 9537					struct vm_area_struct *vma,
 9538					struct perf_addr_filter_range *fr)
 9539{
 9540	unsigned long vma_size = vma->vm_end - vma->vm_start;
 9541	unsigned long off = vma->vm_pgoff << PAGE_SHIFT;
 9542	struct file *file = vma->vm_file;
 9543
 9544	if (!perf_addr_filter_match(filter, file, off, vma_size))
 9545		return false;
 9546
 9547	if (filter->offset < off) {
 9548		fr->start = vma->vm_start;
 9549		fr->size = min(vma_size, filter->size - (off - filter->offset));
 9550	} else {
 9551		fr->start = vma->vm_start + filter->offset - off;
 9552		fr->size = min(vma->vm_end - fr->start, filter->size);
 9553	}
 9554
 9555	return true;
 9556}
 9557
 9558static void __perf_addr_filters_adjust(struct perf_event *event, void *data)
 9559{
 9560	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
 9561	struct vm_area_struct *vma = data;
 9562	struct perf_addr_filter *filter;
 9563	unsigned int restart = 0, count = 0;
 9564	unsigned long flags;
 9565
 9566	if (!has_addr_filter(event))
 9567		return;
 9568
 9569	if (!vma->vm_file)
 9570		return;
 9571
 9572	raw_spin_lock_irqsave(&ifh->lock, flags);
 9573	list_for_each_entry(filter, &ifh->list, entry) {
 9574		if (perf_addr_filter_vma_adjust(filter, vma,
 9575						&event->addr_filter_ranges[count]))
 9576			restart++;
 9577
 9578		count++;
 9579	}
 9580
 9581	if (restart)
 9582		event->addr_filters_gen++;
 9583	raw_spin_unlock_irqrestore(&ifh->lock, flags);
 9584
 9585	if (restart)
 9586		perf_event_stop(event, 1);
 9587}
 9588
 9589/*
 9590 * Adjust all task's events' filters to the new vma
 9591 */
 9592static void perf_addr_filters_adjust(struct vm_area_struct *vma)
 9593{
 9594	struct perf_event_context *ctx;
 9595
 9596	/*
 9597	 * Data tracing isn't supported yet and as such there is no need
 9598	 * to keep track of anything that isn't related to executable code:
 9599	 */
 9600	if (!(vma->vm_flags & VM_EXEC))
 9601		return;
 9602
 9603	rcu_read_lock();
 9604	ctx = rcu_dereference(current->perf_event_ctxp);
 9605	if (ctx)
 9606		perf_iterate_ctx(ctx, __perf_addr_filters_adjust, vma, true);
 9607	rcu_read_unlock();
 9608}
 9609
 9610void perf_event_mmap(struct vm_area_struct *vma)
 9611{
 9612	struct perf_mmap_event mmap_event;
 9613
 9614	if (!atomic_read(&nr_mmap_events))
 9615		return;
 9616
 9617	mmap_event = (struct perf_mmap_event){
 9618		.vma	= vma,
 9619		/* .file_name */
 9620		/* .file_size */
 9621		.event_id  = {
 9622			.header = {
 9623				.type = PERF_RECORD_MMAP,
 9624				.misc = PERF_RECORD_MISC_USER,
 9625				/* .size */
 9626			},
 9627			/* .pid */
 9628			/* .tid */
 9629			.start  = vma->vm_start,
 9630			.len    = vma->vm_end - vma->vm_start,
 9631			.pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
 9632		},
 9633		/* .maj (attr_mmap2 only) */
 9634		/* .min (attr_mmap2 only) */
 9635		/* .ino (attr_mmap2 only) */
 9636		/* .ino_generation (attr_mmap2 only) */
 9637		/* .prot (attr_mmap2 only) */
 9638		/* .flags (attr_mmap2 only) */
 9639	};
 9640
 9641	perf_addr_filters_adjust(vma);
 9642	perf_event_mmap_event(&mmap_event);
 9643}
 9644
 9645void perf_event_aux_event(struct perf_event *event, unsigned long head,
 9646			  unsigned long size, u64 flags)
 9647{
 9648	struct perf_output_handle handle;
 9649	struct perf_sample_data sample;
 9650	struct perf_aux_event {
 9651		struct perf_event_header	header;
 9652		u64				offset;
 9653		u64				size;
 9654		u64				flags;
 9655	} rec = {
 9656		.header = {
 9657			.type = PERF_RECORD_AUX,
 9658			.misc = 0,
 9659			.size = sizeof(rec),
 9660		},
 9661		.offset		= head,
 9662		.size		= size,
 9663		.flags		= flags,
 9664	};
 9665	int ret;
 9666
 9667	perf_event_header__init_id(&rec.header, &sample, event);
 9668	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
 9669
 9670	if (ret)
 9671		return;
 9672
 9673	perf_output_put(&handle, rec);
 9674	perf_event__output_id_sample(event, &handle, &sample);
 9675
 9676	perf_output_end(&handle);
 9677}
 9678
 9679/*
 9680 * Lost/dropped samples logging
 9681 */
 9682void perf_log_lost_samples(struct perf_event *event, u64 lost)
 9683{
 9684	struct perf_output_handle handle;
 9685	struct perf_sample_data sample;
 9686	int ret;
 9687
 9688	struct {
 9689		struct perf_event_header	header;
 9690		u64				lost;
 9691	} lost_samples_event = {
 9692		.header = {
 9693			.type = PERF_RECORD_LOST_SAMPLES,
 9694			.misc = 0,
 9695			.size = sizeof(lost_samples_event),
 9696		},
 9697		.lost		= lost,
 9698	};
 9699
 9700	perf_event_header__init_id(&lost_samples_event.header, &sample, event);
 9701
 9702	ret = perf_output_begin(&handle, &sample, event,
 9703				lost_samples_event.header.size);
 9704	if (ret)
 9705		return;
 9706
 9707	perf_output_put(&handle, lost_samples_event);
 9708	perf_event__output_id_sample(event, &handle, &sample);
 9709	perf_output_end(&handle);
 9710}
 9711
 9712/*
 9713 * context_switch tracking
 9714 */
 9715
 9716struct perf_switch_event {
 9717	struct task_struct	*task;
 9718	struct task_struct	*next_prev;
 9719
 9720	struct {
 9721		struct perf_event_header	header;
 9722		u32				next_prev_pid;
 9723		u32				next_prev_tid;
 9724	} event_id;
 9725};
 9726
 9727static int perf_event_switch_match(struct perf_event *event)
 9728{
 9729	return event->attr.context_switch;
 9730}
 9731
 9732static void perf_event_switch_output(struct perf_event *event, void *data)
 9733{
 9734	struct perf_switch_event *se = data;
 9735	struct perf_output_handle handle;
 9736	struct perf_sample_data sample;
 9737	int ret;
 9738
 9739	if (!perf_event_switch_match(event))
 9740		return;
 9741
 9742	/* Only CPU-wide events are allowed to see next/prev pid/tid */
 9743	if (event->ctx->task) {
 9744		se->event_id.header.type = PERF_RECORD_SWITCH;
 9745		se->event_id.header.size = sizeof(se->event_id.header);
 9746	} else {
 9747		se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE;
 9748		se->event_id.header.size = sizeof(se->event_id);
 9749		se->event_id.next_prev_pid =
 9750					perf_event_pid(event, se->next_prev);
 9751		se->event_id.next_prev_tid =
 9752					perf_event_tid(event, se->next_prev);
 9753	}
 9754
 9755	perf_event_header__init_id(&se->event_id.header, &sample, event);
 9756
 9757	ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size);
 9758	if (ret)
 9759		return;
 9760
 9761	if (event->ctx->task)
 9762		perf_output_put(&handle, se->event_id.header);
 9763	else
 9764		perf_output_put(&handle, se->event_id);
 9765
 9766	perf_event__output_id_sample(event, &handle, &sample);
 9767
 9768	perf_output_end(&handle);
 9769}
 9770
 9771static void perf_event_switch(struct task_struct *task,
 9772			      struct task_struct *next_prev, bool sched_in)
 9773{
 9774	struct perf_switch_event switch_event;
 9775
 9776	/* N.B. caller checks nr_switch_events != 0 */
 9777
 9778	switch_event = (struct perf_switch_event){
 9779		.task		= task,
 9780		.next_prev	= next_prev,
 9781		.event_id	= {
 9782			.header = {
 9783				/* .type */
 9784				.misc = sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT,
 9785				/* .size */
 9786			},
 9787			/* .next_prev_pid */
 9788			/* .next_prev_tid */
 9789		},
 9790	};
 9791
 9792	if (!sched_in && task_is_runnable(task)) {
 9793		switch_event.event_id.header.misc |=
 9794				PERF_RECORD_MISC_SWITCH_OUT_PREEMPT;
 9795	}
 9796
 9797	perf_iterate_sb(perf_event_switch_output, &switch_event, NULL);
 9798}
 9799
 9800/*
 9801 * IRQ throttle logging
 9802 */
 9803
 9804static void perf_log_throttle(struct perf_event *event, int enable)
 9805{
 9806	struct perf_output_handle handle;
 9807	struct perf_sample_data sample;
 9808	int ret;
 9809
 9810	struct {
 9811		struct perf_event_header	header;
 9812		u64				time;
 9813		u64				id;
 9814		u64				stream_id;
 9815	} throttle_event = {
 9816		.header = {
 9817			.type = PERF_RECORD_THROTTLE,
 9818			.misc = 0,
 9819			.size = sizeof(throttle_event),
 9820		},
 9821		.time		= perf_event_clock(event),
 9822		.id		= primary_event_id(event),
 9823		.stream_id	= event->id,
 9824	};
 9825
 9826	if (enable)
 9827		throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
 9828
 9829	perf_event_header__init_id(&throttle_event.header, &sample, event);
 9830
 9831	ret = perf_output_begin(&handle, &sample, event,
 9832				throttle_event.header.size);
 9833	if (ret)
 9834		return;
 9835
 9836	perf_output_put(&handle, throttle_event);
 9837	perf_event__output_id_sample(event, &handle, &sample);
 9838	perf_output_end(&handle);
 9839}
 9840
 9841/*
 9842 * ksymbol register/unregister tracking
 9843 */
 9844
 9845struct perf_ksymbol_event {
 9846	const char	*name;
 9847	int		name_len;
 9848	struct {
 9849		struct perf_event_header        header;
 9850		u64				addr;
 9851		u32				len;
 9852		u16				ksym_type;
 9853		u16				flags;
 9854	} event_id;
 9855};
 9856
 9857static int perf_event_ksymbol_match(struct perf_event *event)
 9858{
 9859	return event->attr.ksymbol;
 9860}
 9861
 9862static void perf_event_ksymbol_output(struct perf_event *event, void *data)
 9863{
 9864	struct perf_ksymbol_event *ksymbol_event = data;
 9865	struct perf_output_handle handle;
 9866	struct perf_sample_data sample;
 9867	int ret;
 9868
 9869	if (!perf_event_ksymbol_match(event))
 9870		return;
 9871
 9872	perf_event_header__init_id(&ksymbol_event->event_id.header,
 9873				   &sample, event);
 9874	ret = perf_output_begin(&handle, &sample, event,
 9875				ksymbol_event->event_id.header.size);
 9876	if (ret)
 9877		return;
 9878
 9879	perf_output_put(&handle, ksymbol_event->event_id);
 9880	__output_copy(&handle, ksymbol_event->name, ksymbol_event->name_len);
 9881	perf_event__output_id_sample(event, &handle, &sample);
 9882
 9883	perf_output_end(&handle);
 9884}
 9885
 9886void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len, bool unregister,
 9887			const char *sym)
 9888{
 9889	struct perf_ksymbol_event ksymbol_event;
 9890	char name[KSYM_NAME_LEN];
 9891	u16 flags = 0;
 9892	int name_len;
 9893
 9894	if (!atomic_read(&nr_ksymbol_events))
 9895		return;
 9896
 9897	if (ksym_type >= PERF_RECORD_KSYMBOL_TYPE_MAX ||
 9898	    ksym_type == PERF_RECORD_KSYMBOL_TYPE_UNKNOWN)
 9899		goto err;
 9900
 9901	strscpy(name, sym);
 9902	name_len = strlen(name) + 1;
 9903	while (!IS_ALIGNED(name_len, sizeof(u64)))
 9904		name[name_len++] = '\0';
 9905	BUILD_BUG_ON(KSYM_NAME_LEN % sizeof(u64));
 9906
 9907	if (unregister)
 9908		flags |= PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER;
 9909
 9910	ksymbol_event = (struct perf_ksymbol_event){
 9911		.name = name,
 9912		.name_len = name_len,
 9913		.event_id = {
 9914			.header = {
 9915				.type = PERF_RECORD_KSYMBOL,
 9916				.size = sizeof(ksymbol_event.event_id) +
 9917					name_len,
 9918			},
 9919			.addr = addr,
 9920			.len = len,
 9921			.ksym_type = ksym_type,
 9922			.flags = flags,
 9923		},
 9924	};
 9925
 9926	perf_iterate_sb(perf_event_ksymbol_output, &ksymbol_event, NULL);
 9927	return;
 9928err:
 9929	WARN_ONCE(1, "%s: Invalid KSYMBOL type 0x%x\n", __func__, ksym_type);
 9930}
 9931
 9932/*
 9933 * bpf program load/unload tracking
 9934 */
 9935
 9936struct perf_bpf_event {
 9937	struct bpf_prog	*prog;
 9938	struct {
 9939		struct perf_event_header        header;
 9940		u16				type;
 9941		u16				flags;
 9942		u32				id;
 9943		u8				tag[BPF_TAG_SIZE];
 9944	} event_id;
 9945};
 9946
 9947static int perf_event_bpf_match(struct perf_event *event)
 9948{
 9949	return event->attr.bpf_event;
 9950}
 9951
 9952static void perf_event_bpf_output(struct perf_event *event, void *data)
 9953{
 9954	struct perf_bpf_event *bpf_event = data;
 9955	struct perf_output_handle handle;
 9956	struct perf_sample_data sample;
 9957	int ret;
 9958
 9959	if (!perf_event_bpf_match(event))
 9960		return;
 9961
 9962	perf_event_header__init_id(&bpf_event->event_id.header,
 9963				   &sample, event);
 9964	ret = perf_output_begin(&handle, &sample, event,
 9965				bpf_event->event_id.header.size);
 9966	if (ret)
 9967		return;
 9968
 9969	perf_output_put(&handle, bpf_event->event_id);
 9970	perf_event__output_id_sample(event, &handle, &sample);
 9971
 9972	perf_output_end(&handle);
 9973}
 9974
 9975static void perf_event_bpf_emit_ksymbols(struct bpf_prog *prog,
 9976					 enum perf_bpf_event_type type)
 9977{
 9978	bool unregister = type == PERF_BPF_EVENT_PROG_UNLOAD;
 9979	int i;
 9980
 9981	perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF,
 9982			   (u64)(unsigned long)prog->bpf_func,
 9983			   prog->jited_len, unregister,
 9984			   prog->aux->ksym.name);
 9985
 9986	for (i = 1; i < prog->aux->func_cnt; i++) {
 9987		struct bpf_prog *subprog = prog->aux->func[i];
 9988
 9989		perf_event_ksymbol(
 9990			PERF_RECORD_KSYMBOL_TYPE_BPF,
 9991			(u64)(unsigned long)subprog->bpf_func,
 9992			subprog->jited_len, unregister,
 9993			subprog->aux->ksym.name);
 9994	}
 9995}
 9996
 9997void perf_event_bpf_event(struct bpf_prog *prog,
 9998			  enum perf_bpf_event_type type,
 9999			  u16 flags)
10000{
10001	struct perf_bpf_event bpf_event;
10002
10003	switch (type) {
10004	case PERF_BPF_EVENT_PROG_LOAD:
10005	case PERF_BPF_EVENT_PROG_UNLOAD:
10006		if (atomic_read(&nr_ksymbol_events))
10007			perf_event_bpf_emit_ksymbols(prog, type);
10008		break;
10009	default:
10010		return;
10011	}
10012
10013	if (!atomic_read(&nr_bpf_events))
10014		return;
10015
10016	bpf_event = (struct perf_bpf_event){
10017		.prog = prog,
10018		.event_id = {
10019			.header = {
10020				.type = PERF_RECORD_BPF_EVENT,
10021				.size = sizeof(bpf_event.event_id),
10022			},
10023			.type = type,
10024			.flags = flags,
10025			.id = prog->aux->id,
10026		},
10027	};
10028
10029	BUILD_BUG_ON(BPF_TAG_SIZE % sizeof(u64));
10030
10031	memcpy(bpf_event.event_id.tag, prog->tag, BPF_TAG_SIZE);
10032	perf_iterate_sb(perf_event_bpf_output, &bpf_event, NULL);
10033}
10034
10035struct perf_callchain_deferred_event {
10036	struct unwind_stacktrace *trace;
10037	struct {
10038		struct perf_event_header	header;
10039		u64				cookie;
10040		u64				nr;
10041		u64				ips[];
10042	} event;
10043};
10044
10045static void perf_callchain_deferred_output(struct perf_event *event, void *data)
10046{
10047	struct perf_callchain_deferred_event *deferred_event = data;
10048	struct perf_output_handle handle;
10049	struct perf_sample_data sample;
10050	int ret, size = deferred_event->event.header.size;
10051
10052	if (!event->attr.defer_output)
10053		return;
10054
10055	/* XXX do we really need sample_id_all for this ??? */
10056	perf_event_header__init_id(&deferred_event->event.header, &sample, event);
10057
10058	ret = perf_output_begin(&handle, &sample, event,
10059				deferred_event->event.header.size);
10060	if (ret)
10061		goto out;
10062
10063	perf_output_put(&handle, deferred_event->event);
10064	for (int i = 0; i < deferred_event->trace->nr; i++) {
10065		u64 entry = deferred_event->trace->entries[i];
10066		perf_output_put(&handle, entry);
10067	}
10068	perf_event__output_id_sample(event, &handle, &sample);
10069
10070	perf_output_end(&handle);
10071out:
10072	deferred_event->event.header.size = size;
10073}
10074
10075static void perf_unwind_deferred_callback(struct unwind_work *work,
10076					 struct unwind_stacktrace *trace, u64 cookie)
10077{
10078	struct perf_callchain_deferred_event deferred_event = {
10079		.trace = trace,
10080		.event = {
10081			.header = {
10082				.type = PERF_RECORD_CALLCHAIN_DEFERRED,
10083				.misc = PERF_RECORD_MISC_USER,
10084				.size = sizeof(deferred_event.event) +
10085					(trace->nr * sizeof(u64)),
10086			},
10087			.cookie = cookie,
10088			.nr = trace->nr,
10089		},
10090	};
10091
10092	perf_iterate_sb(perf_callchain_deferred_output, &deferred_event, NULL);
10093}
10094
10095struct perf_text_poke_event {
10096	const void		*old_bytes;
10097	const void		*new_bytes;
10098	size_t			pad;
10099	u16			old_len;
10100	u16			new_len;
10101
10102	struct {
10103		struct perf_event_header	header;
10104
10105		u64				addr;
10106	} event_id;
10107};
10108
10109static int perf_event_text_poke_match(struct perf_event *event)
10110{
10111	return event->attr.text_poke;
10112}
10113
10114static void perf_event_text_poke_output(struct perf_event *event, void *data)
10115{
10116	struct perf_text_poke_event *text_poke_event = data;
10117	struct perf_output_handle handle;
10118	struct perf_sample_data sample;
10119	u64 padding = 0;
10120	int ret;
10121
10122	if (!perf_event_text_poke_match(event))
10123		return;
10124
10125	perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event);
10126
10127	ret = perf_output_begin(&handle, &sample, event,
10128				text_poke_event->event_id.header.size);
10129	if (ret)
10130		return;
10131
10132	perf_output_put(&handle, text_poke_event->event_id);
10133	perf_output_put(&handle, text_poke_event->old_len);
10134	perf_output_put(&handle, text_poke_event->new_len);
10135
10136	__output_copy(&handle, text_poke_event->old_bytes, text_poke_event->old_len);
10137	__output_copy(&handle, text_poke_event->new_bytes, text_poke_event->new_len);
10138
10139	if (text_poke_event->pad)
10140		__output_copy(&handle, &padding, text_poke_event->pad);
10141
10142	perf_event__output_id_sample(event, &handle, &sample);
10143
10144	perf_output_end(&handle);
10145}
10146
10147void perf_event_text_poke(const void *addr, const void *old_bytes,
10148			  size_t old_len, const void *new_bytes, size_t new_len)
10149{
10150	struct perf_text_poke_event text_poke_event;
10151	size_t tot, pad;
10152
10153	if (!atomic_read(&nr_text_poke_events))
10154		return;
10155
10156	tot  = sizeof(text_poke_event.old_len) + old_len;
10157	tot += sizeof(text_poke_event.new_len) + new_len;
10158	pad  = ALIGN(tot, sizeof(u64)) - tot;
10159
10160	text_poke_event = (struct perf_text_poke_event){
10161		.old_bytes    = old_bytes,
10162		.new_bytes    = new_bytes,
10163		.pad          = pad,
10164		.old_len      = old_len,
10165		.new_len      = new_len,
10166		.event_id  = {
10167			.header = {
10168				.type = PERF_RECORD_TEXT_POKE,
10169				.misc = PERF_RECORD_MISC_KERNEL,
10170				.size = sizeof(text_poke_event.event_id) + tot + pad,
10171			},
10172			.addr = (unsigned long)addr,
10173		},
10174	};
10175
10176	perf_iterate_sb(perf_event_text_poke_output, &text_poke_event, NULL);
10177}
10178
10179void perf_event_itrace_started(struct perf_event *event)
10180{
10181	WRITE_ONCE(event->attach_state, event->attach_state | PERF_ATTACH_ITRACE);
10182}
10183
10184static void perf_log_itrace_start(struct perf_event *event)
10185{
10186	struct perf_output_handle handle;
10187	struct perf_sample_data sample;
10188	struct perf_aux_event {
10189		struct perf_event_header        header;
10190		u32				pid;
10191		u32				tid;
10192	} rec;
10193	int ret;
10194
10195	if (event->parent)
10196		event = event->parent;
10197
10198	if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) ||
10199	    event->attach_state & PERF_ATTACH_ITRACE)
10200		return;
10201
10202	rec.header.type	= PERF_RECORD_ITRACE_START;
10203	rec.header.misc	= 0;
10204	rec.header.size	= sizeof(rec);
10205	rec.pid	= perf_event_pid(event, current);
10206	rec.tid	= perf_event_tid(event, current);
10207
10208	perf_event_header__init_id(&rec.header, &sample, event);
10209	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
10210
10211	if (ret)
10212		return;
10213
10214	perf_output_put(&handle, rec);
10215	perf_event__output_id_sample(event, &handle, &sample);
10216
10217	perf_output_end(&handle);
10218}
10219
10220void perf_report_aux_output_id(struct perf_event *event, u64 hw_id)
10221{
10222	struct perf_output_handle handle;
10223	struct perf_sample_data sample;
10224	struct perf_aux_event {
10225		struct perf_event_header        header;
10226		u64				hw_id;
10227	} rec;
10228	int ret;
10229
10230	if (event->parent)
10231		event = event->parent;
10232
10233	rec.header.type	= PERF_RECORD_AUX_OUTPUT_HW_ID;
10234	rec.header.misc	= 0;
10235	rec.header.size	= sizeof(rec);
10236	rec.hw_id	= hw_id;
10237
10238	perf_event_header__init_id(&rec.header, &sample, event);
10239	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
10240
10241	if (ret)
10242		return;
10243
10244	perf_output_put(&handle, rec);
10245	perf_event__output_id_sample(event, &handle, &sample);
10246
10247	perf_output_end(&handle);
10248}
10249EXPORT_SYMBOL_GPL(perf_report_aux_output_id);
10250
10251static int
10252__perf_event_account_interrupt(struct perf_event *event, int throttle)
10253{
10254	struct hw_perf_event *hwc = &event->hw;
10255	int ret = 0;
10256	u64 seq;
10257
10258	seq = __this_cpu_read(perf_throttled_seq);
10259	if (seq != hwc->interrupts_seq) {
10260		hwc->interrupts_seq = seq;
10261		hwc->interrupts = 1;
10262	} else {
10263		hwc->interrupts++;
10264	}
10265
10266	if (unlikely(throttle && hwc->interrupts >= max_samples_per_tick)) {
10267		__this_cpu_inc(perf_throttled_count);
10268		tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
10269		perf_event_throttle_group(event);
10270		ret = 1;
10271	}
10272
10273	if (event->attr.freq) {
10274		u64 now = perf_clock();
10275		s64 delta = now - hwc->freq_time_stamp;
10276
10277		hwc->freq_time_stamp = now;
10278
10279		if (delta > 0 && delta < 2*TICK_NSEC)
10280			perf_adjust_period(event, delta, hwc->last_period, true);
10281	}
10282
10283	return ret;
10284}
10285
10286int perf_event_account_interrupt(struct perf_event *event)
10287{
10288	return __perf_event_account_interrupt(event, 1);
10289}
10290
10291static inline bool sample_is_allowed(struct perf_event *event, struct pt_regs *regs)
10292{
10293	/*
10294	 * Due to interrupt latency (AKA "skid"), we may enter the
10295	 * kernel before taking an overflow, even if the PMU is only
10296	 * counting user events.
10297	 */
10298	if (event->attr.exclude_kernel && !user_mode(regs))
10299		return false;
10300
10301	return true;
10302}
10303
10304#ifdef CONFIG_BPF_SYSCALL
10305static int bpf_overflow_handler(struct perf_event *event,
10306				struct perf_sample_data *data,
10307				struct pt_regs *regs)
10308{
10309	struct bpf_perf_event_data_kern ctx = {
10310		.data = data,
10311		.event = event,
10312	};
10313	struct bpf_prog *prog;
10314	int ret = 0;
10315
10316	ctx.regs = perf_arch_bpf_user_pt_regs(regs);
10317	if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1))
10318		goto out;
10319	rcu_read_lock();
10320	prog = READ_ONCE(event->prog);
10321	if (prog) {
10322		perf_prepare_sample(data, event, regs);
10323		ret = bpf_prog_run(prog, &ctx);
10324	}
10325	rcu_read_unlock();
10326out:
10327	__this_cpu_dec(bpf_prog_active);
10328
10329	return ret;
10330}
10331
10332static inline int perf_event_set_bpf_handler(struct perf_event *event,
10333					     struct bpf_prog *prog,
10334					     u64 bpf_cookie)
10335{
10336	if (event->overflow_handler_context)
10337		/* hw breakpoint or kernel counter */
10338		return -EINVAL;
10339
10340	if (event->prog)
10341		return -EEXIST;
10342
10343	if (prog->type != BPF_PROG_TYPE_PERF_EVENT)
10344		return -EINVAL;
10345
10346	if (event->attr.precise_ip &&
10347	    prog->call_get_stack &&
10348	    (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) ||
10349	     event->attr.exclude_callchain_kernel ||
10350	     event->attr.exclude_callchain_user)) {
10351		/*
10352		 * On perf_event with precise_ip, calling bpf_get_stack()
10353		 * may trigger unwinder warnings and occasional crashes.
10354		 * bpf_get_[stack|stackid] works around this issue by using
10355		 * callchain attached to perf_sample_data. If the
10356		 * perf_event does not full (kernel and user) callchain
10357		 * attached to perf_sample_data, do not allow attaching BPF
10358		 * program that calls bpf_get_[stack|stackid].
10359		 */
10360		return -EPROTO;
10361	}
10362
10363	event->prog = prog;
10364	event->bpf_cookie = bpf_cookie;
10365	return 0;
10366}
10367
10368static inline void perf_event_free_bpf_handler(struct perf_event *event)
10369{
10370	struct bpf_prog *prog = event->prog;
10371
10372	if (!prog)
10373		return;
10374
10375	event->prog = NULL;
10376	bpf_prog_put(prog);
10377}
10378#else
10379static inline int bpf_overflow_handler(struct perf_event *event,
10380				       struct perf_sample_data *data,
10381				       struct pt_regs *regs)
10382{
10383	return 1;
10384}
10385
10386static inline int perf_event_set_bpf_handler(struct perf_event *event,
10387					     struct bpf_prog *prog,
10388					     u64 bpf_cookie)
10389{
10390	return -EOPNOTSUPP;
10391}
10392
10393static inline void perf_event_free_bpf_handler(struct perf_event *event)
10394{
10395}
10396#endif
10397
10398/*
10399 * Generic event overflow handling, sampling.
10400 */
10401
10402static int __perf_event_overflow(struct perf_event *event,
10403				 int throttle, struct perf_sample_data *data,
10404				 struct pt_regs *regs)
10405{
10406	int events = atomic_read(&event->event_limit);
10407	int ret = 0;
10408
10409	/*
10410	 * Non-sampling counters might still use the PMI to fold short
10411	 * hardware counters, ignore those.
10412	 */
10413	if (unlikely(!is_sampling_event(event)))
10414		return 0;
10415
10416	ret = __perf_event_account_interrupt(event, throttle);
10417
10418	if (event->attr.aux_pause)
10419		perf_event_aux_pause(event->aux_event, true);
10420
10421	if (event->prog && event->prog->type == BPF_PROG_TYPE_PERF_EVENT &&
10422	    !bpf_overflow_handler(event, data, regs))
10423		goto out;
10424
10425	/*
10426	 * XXX event_limit might not quite work as expected on inherited
10427	 * events
10428	 */
10429
10430	event->pending_kill = POLL_IN;
10431	if (events && atomic_dec_and_test(&event->event_limit)) {
10432		ret = 1;
10433		event->pending_kill = POLL_HUP;
10434		perf_event_disable_inatomic(event);
10435		event->pmu->stop(event, 0);
10436	}
10437
10438	if (event->attr.sigtrap) {
10439		/*
10440		 * The desired behaviour of sigtrap vs invalid samples is a bit
10441		 * tricky; on the one hand, one should not loose the SIGTRAP if
10442		 * it is the first event, on the other hand, we should also not
10443		 * trigger the WARN or override the data address.
10444		 */
10445		bool valid_sample = sample_is_allowed(event, regs);
10446		unsigned int pending_id = 1;
10447		enum task_work_notify_mode notify_mode;
10448
10449		if (regs)
10450			pending_id = hash32_ptr((void *)instruction_pointer(regs)) ?: 1;
10451
10452		notify_mode = in_nmi() ? TWA_NMI_CURRENT : TWA_RESUME;
10453
10454		if (!event->pending_work &&
10455		    !task_work_add(current, &event->pending_task, notify_mode)) {
10456			event->pending_work = pending_id;
10457			local_inc(&event->ctx->nr_no_switch_fast);
10458			WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount));
10459
10460			event->pending_addr = 0;
10461			if (valid_sample && (data->sample_flags & PERF_SAMPLE_ADDR))
10462				event->pending_addr = data->addr;
10463
10464		} else if (event->attr.exclude_kernel && valid_sample) {
10465			/*
10466			 * Should not be able to return to user space without
10467			 * consuming pending_work; with exceptions:
10468			 *
10469			 *  1. Where !exclude_kernel, events can overflow again
10470			 *     in the kernel without returning to user space.
10471			 *
10472			 *  2. Events that can overflow again before the IRQ-
10473			 *     work without user space progress (e.g. hrtimer).
10474			 *     To approximate progress (with false negatives),
10475			 *     check 32-bit hash of the current IP.
10476			 */
10477			WARN_ON_ONCE(event->pending_work != pending_id);
10478		}
10479	}
10480
10481	READ_ONCE(event->overflow_handler)(event, data, regs);
10482
10483	if (*perf_event_fasync(event) && event->pending_kill) {
10484		event->pending_wakeup = 1;
10485		irq_work_queue(&event->pending_irq);
10486	}
10487out:
10488	if (event->attr.aux_resume)
10489		perf_event_aux_pause(event->aux_event, false);
10490
10491	return ret;
10492}
10493
10494int perf_event_overflow(struct perf_event *event,
10495			struct perf_sample_data *data,
10496			struct pt_regs *regs)
10497{
10498	/*
10499	 * Entry point from hardware PMI, interrupts should be disabled here.
10500	 * This serializes us against perf_event_remove_from_context() in
10501	 * things like perf_event_release_kernel().
10502	 */
10503	lockdep_assert_irqs_disabled();
10504
10505	return __perf_event_overflow(event, 1, data, regs);
10506}
10507
10508/*
10509 * Generic software event infrastructure
10510 */
10511
10512struct swevent_htable {
10513	struct swevent_hlist		*swevent_hlist;
10514	struct mutex			hlist_mutex;
10515	int				hlist_refcount;
10516};
10517static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
10518
10519/*
10520 * We directly increment event->count and keep a second value in
10521 * event->hw.period_left to count intervals. This period event
10522 * is kept in the range [-sample_period, 0] so that we can use the
10523 * sign as trigger.
10524 */
10525
10526u64 perf_swevent_set_period(struct perf_event *event)
10527{
10528	struct hw_perf_event *hwc = &event->hw;
10529	u64 period = hwc->last_period;
10530	u64 nr, offset;
10531	s64 old, val;
10532
10533	hwc->last_period = hwc->sample_period;
10534
10535	old = local64_read(&hwc->period_left);
10536	do {
10537		val = old;
10538		if (val < 0)
10539			return 0;
10540
10541		nr = div64_u64(period + val, period);
10542		offset = nr * period;
10543		val -= offset;
10544	} while (!local64_try_cmpxchg(&hwc->period_left, &old, val));
10545
10546	return nr;
10547}
10548
10549static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
10550				    struct perf_sample_data *data,
10551				    struct pt_regs *regs)
10552{
10553	struct hw_perf_event *hwc = &event->hw;
10554	int throttle = 0;
10555
10556	if (!overflow)
10557		overflow = perf_swevent_set_period(event);
10558
10559	if (hwc->interrupts == MAX_INTERRUPTS)
10560		return;
10561
10562	for (; overflow; overflow--) {
10563		if (__perf_event_overflow(event, throttle,
10564					    data, regs)) {
10565			/*
10566			 * We inhibit the overflow from happening when
10567			 * hwc->interrupts == MAX_INTERRUPTS.
10568			 */
10569			break;
10570		}
10571		throttle = 1;
10572	}
10573}
10574
10575static void perf_swevent_event(struct perf_event *event, u64 nr,
10576			       struct perf_sample_data *data,
10577			       struct pt_regs *regs)
10578{
10579	struct hw_perf_event *hwc = &event->hw;
10580
10581	/*
10582	 * This is:
10583	 *   - software		preempt
10584	 *   - tracepoint	preempt
10585	 *   -   tp_target_task	irq (ctx->lock)
10586	 *   - uprobes		preempt/irq
10587	 *   - kprobes		preempt/irq
10588	 *   - hw_breakpoint	irq
10589	 *
10590	 * Any of these are sufficient to hold off RCU and thus ensure @event
10591	 * exists.
10592	 */
10593	lockdep_assert_preemption_disabled();
10594	local64_add(nr, &event->count);
10595
10596	if (!regs)
10597		return;
10598
10599	if (!is_sampling_event(event))
10600		return;
10601
10602	/*
10603	 * Serialize against event_function_call() IPIs like normal overflow
10604	 * event handling. Specifically, must not allow
10605	 * perf_event_release_kernel() -> perf_remove_from_context() to make
10606	 * progress and 'release' the event from under us.
10607	 */
10608	guard(irqsave)();
10609	if (event->state != PERF_EVENT_STATE_ACTIVE)
10610		return;
10611
10612	if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
10613		data->period = nr;
10614		return perf_swevent_overflow(event, 1, data, regs);
10615	} else
10616		data->period = event->hw.last_period;
10617
10618	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
10619		return perf_swevent_overflow(event, 1, data, regs);
10620
10621	if (local64_add_negative(nr, &hwc->period_left))
10622		return;
10623
10624	perf_swevent_overflow(event, 0, data, regs);
10625}
10626
10627int perf_exclude_event(struct perf_event *event, struct pt_regs *regs)
10628{
10629	if (event->hw.state & PERF_HES_STOPPED)
10630		return 1;
10631
10632	if (regs) {
10633		if (event->attr.exclude_user && user_mode(regs))
10634			return 1;
10635
10636		if (event->attr.exclude_kernel && !user_mode(regs))
10637			return 1;
10638	}
10639
10640	return 0;
10641}
10642
10643static int perf_swevent_match(struct perf_event *event,
10644				enum perf_type_id type,
10645				u32 event_id,
10646				struct perf_sample_data *data,
10647				struct pt_regs *regs)
10648{
10649	if (event->attr.type != type)
10650		return 0;
10651
10652	if (event->attr.config != event_id)
10653		return 0;
10654
10655	if (perf_exclude_event(event, regs))
10656		return 0;
10657
10658	return 1;
10659}
10660
10661static inline u64 swevent_hash(u64 type, u32 event_id)
10662{
10663	u64 val = event_id | (type << 32);
10664
10665	return hash_64(val, SWEVENT_HLIST_BITS);
10666}
10667
10668static inline struct hlist_head *
10669__find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
10670{
10671	u64 hash = swevent_hash(type, event_id);
10672
10673	return &hlist->heads[hash];
10674}
10675
10676/* For the read side: events when they trigger */
10677static inline struct hlist_head *
10678find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
10679{
10680	struct swevent_hlist *hlist;
10681
10682	hlist = rcu_dereference(swhash->swevent_hlist);
10683	if (!hlist)
10684		return NULL;
10685
10686	return __find_swevent_head(hlist, type, event_id);
10687}
10688
10689/* For the event head insertion and removal in the hlist */
10690static inline struct hlist_head *
10691find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
10692{
10693	struct swevent_hlist *hlist;
10694	u32 event_id = event->attr.config;
10695	u64 type = event->attr.type;
10696
10697	/*
10698	 * Event scheduling is always serialized against hlist allocation
10699	 * and release. Which makes the protected version suitable here.
10700	 * The context lock guarantees that.
10701	 */
10702	hlist = rcu_dereference_protected(swhash->swevent_hlist,
10703					  lockdep_is_held(&event->ctx->lock));
10704	if (!hlist)
10705		return NULL;
10706
10707	return __find_swevent_head(hlist, type, event_id);
10708}
10709
10710static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
10711				    u64 nr,
10712				    struct perf_sample_data *data,
10713				    struct pt_regs *regs)
10714{
10715	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
10716	struct perf_event *event;
10717	struct hlist_head *head;
10718
10719	rcu_read_lock();
10720	head = find_swevent_head_rcu(swhash, type, event_id);
10721	if (!head)
10722		goto end;
10723
10724	hlist_for_each_entry_rcu(event, head, hlist_entry) {
10725		if (perf_swevent_match(event, type, event_id, data, regs))
10726			perf_swevent_event(event, nr, data, regs);
10727	}
10728end:
10729	rcu_read_unlock();
10730}
10731
10732DEFINE_PER_CPU(struct pt_regs, __perf_regs[4]);
10733
10734int perf_swevent_get_recursion_context(void)
10735{
10736	return get_recursion_context(current->perf_recursion);
10737}
10738EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
10739
10740void perf_swevent_put_recursion_context(int rctx)
10741{
10742	put_recursion_context(current->perf_recursion, rctx);
10743}
10744
10745void ___perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
10746{
10747	struct perf_sample_data data;
10748
10749	if (WARN_ON_ONCE(!regs))
10750		return;
10751
10752	perf_sample_data_init(&data, addr, 0);
10753	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
10754}
10755
10756void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
10757{
10758	int rctx;
10759
10760	preempt_disable_notrace();
10761	rctx = perf_swevent_get_recursion_context();
10762	if (unlikely(rctx < 0))
10763		goto fail;
10764
10765	___perf_sw_event(event_id, nr, regs, addr);
10766
10767	perf_swevent_put_recursion_context(rctx);
10768fail:
10769	preempt_enable_notrace();
10770}
10771
10772static void perf_swevent_read(struct perf_event *event)
10773{
10774}
10775
10776static int perf_swevent_add(struct perf_event *event, int flags)
10777{
10778	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
10779	struct hw_perf_event *hwc = &event->hw;
10780	struct hlist_head *head;
10781
10782	if (is_sampling_event(event)) {
10783		hwc->last_period = hwc->sample_period;
10784		perf_swevent_set_period(event);
10785	}
10786
10787	hwc->state = !(flags & PERF_EF_START);
10788
10789	head = find_swevent_head(swhash, event);
10790	if (WARN_ON_ONCE(!head))
10791		return -EINVAL;
10792
10793	hlist_add_head_rcu(&event->hlist_entry, head);
10794	perf_event_update_userpage(event);
10795
10796	return 0;
10797}
10798
10799static void perf_swevent_del(struct perf_event *event, int flags)
10800{
10801	hlist_del_rcu(&event->hlist_entry);
10802}
10803
10804static void perf_swevent_start(struct perf_event *event, int flags)
10805{
10806	event->hw.state = 0;
10807}
10808
10809static void perf_swevent_stop(struct perf_event *event, int flags)
10810{
10811	event->hw.state = PERF_HES_STOPPED;
10812}
10813
10814/* Deref the hlist from the update side */
10815static inline struct swevent_hlist *
10816swevent_hlist_deref(struct swevent_htable *swhash)
10817{
10818	return rcu_dereference_protected(swhash->swevent_hlist,
10819					 lockdep_is_held(&swhash->hlist_mutex));
10820}
10821
10822static void swevent_hlist_release(struct swevent_htable *swhash)
10823{
10824	struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
10825
10826	if (!hlist)
10827		return;
10828
10829	RCU_INIT_POINTER(swhash->swevent_hlist, NULL);
10830	kfree_rcu(hlist, rcu_head);
10831}
10832
10833static void swevent_hlist_put_cpu(int cpu)
10834{
10835	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
10836
10837	mutex_lock(&swhash->hlist_mutex);
10838
10839	if (!--swhash->hlist_refcount)
10840		swevent_hlist_release(swhash);
10841
10842	mutex_unlock(&swhash->hlist_mutex);
10843}
10844
10845static void swevent_hlist_put(void)
10846{
10847	int cpu;
10848
10849	for_each_possible_cpu(cpu)
10850		swevent_hlist_put_cpu(cpu);
10851}
10852
10853static int swevent_hlist_get_cpu(int cpu)
10854{
10855	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
10856	int err = 0;
10857
10858	mutex_lock(&swhash->hlist_mutex);
10859	if (!swevent_hlist_deref(swhash) &&
10860	    cpumask_test_cpu(cpu, perf_online_mask)) {
10861		struct swevent_hlist *hlist;
10862
10863		hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
10864		if (!hlist) {
10865			err = -ENOMEM;
10866			goto exit;
10867		}
10868		rcu_assign_pointer(swhash->swevent_hlist, hlist);
10869	}
10870	swhash->hlist_refcount++;
10871exit:
10872	mutex_unlock(&swhash->hlist_mutex);
10873
10874	return err;
10875}
10876
10877static int swevent_hlist_get(void)
10878{
10879	int err, cpu, failed_cpu;
10880
10881	mutex_lock(&pmus_lock);
10882	for_each_possible_cpu(cpu) {
10883		err = swevent_hlist_get_cpu(cpu);
10884		if (err) {
10885			failed_cpu = cpu;
10886			goto fail;
10887		}
10888	}
10889	mutex_unlock(&pmus_lock);
10890	return 0;
10891fail:
10892	for_each_possible_cpu(cpu) {
10893		if (cpu == failed_cpu)
10894			break;
10895		swevent_hlist_put_cpu(cpu);
10896	}
10897	mutex_unlock(&pmus_lock);
10898	return err;
10899}
10900
10901struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
10902
10903static void sw_perf_event_destroy(struct perf_event *event)
10904{
10905	u64 event_id = event->attr.config;
10906
10907	WARN_ON(event->parent);
10908
10909	static_key_slow_dec(&perf_swevent_enabled[event_id]);
10910	swevent_hlist_put();
10911}
10912
10913static struct pmu perf_cpu_clock; /* fwd declaration */
10914static struct pmu perf_task_clock;
10915
10916static int perf_swevent_init(struct perf_event *event)
10917{
10918	u64 event_id = event->attr.config;
10919
10920	if (event->attr.type != PERF_TYPE_SOFTWARE)
10921		return -ENOENT;
10922
10923	/*
10924	 * no branch sampling for software events
10925	 */
10926	if (has_branch_stack(event))
10927		return -EOPNOTSUPP;
10928
10929	switch (event_id) {
10930	case PERF_COUNT_SW_CPU_CLOCK:
10931		event->attr.type = perf_cpu_clock.type;
10932		return -ENOENT;
10933	case PERF_COUNT_SW_TASK_CLOCK:
10934		event->attr.type = perf_task_clock.type;
10935		return -ENOENT;
10936
10937	default:
10938		break;
10939	}
10940
10941	if (event_id >= PERF_COUNT_SW_MAX)
10942		return -ENOENT;
10943
10944	if (!event->parent) {
10945		int err;
10946
10947		err = swevent_hlist_get();
10948		if (err)
10949			return err;
10950
10951		static_key_slow_inc(&perf_swevent_enabled[event_id]);
10952		event->destroy = sw_perf_event_destroy;
10953	}
10954
10955	return 0;
10956}
10957
10958static struct pmu perf_swevent = {
10959	.task_ctx_nr	= perf_sw_context,
10960
10961	.capabilities	= PERF_PMU_CAP_NO_NMI,
10962
10963	.event_init	= perf_swevent_init,
10964	.add		= perf_swevent_add,
10965	.del		= perf_swevent_del,
10966	.start		= perf_swevent_start,
10967	.stop		= perf_swevent_stop,
10968	.read		= perf_swevent_read,
10969};
10970
10971#ifdef CONFIG_EVENT_TRACING
10972
10973static void tp_perf_event_destroy(struct perf_event *event)
10974{
10975	perf_trace_destroy(event);
10976}
10977
10978static int perf_tp_event_init(struct perf_event *event)
10979{
10980	int err;
10981
10982	if (event->attr.type != PERF_TYPE_TRACEPOINT)
10983		return -ENOENT;
10984
10985	/*
10986	 * no branch sampling for tracepoint events
10987	 */
10988	if (has_branch_stack(event))
10989		return -EOPNOTSUPP;
10990
10991	err = perf_trace_init(event);
10992	if (err)
10993		return err;
10994
10995	event->destroy = tp_perf_event_destroy;
10996
10997	return 0;
10998}
10999
11000static struct pmu perf_tracepoint = {
11001	.task_ctx_nr	= perf_sw_context,
11002
11003	.event_init	= perf_tp_event_init,
11004	.add		= perf_trace_add,
11005	.del		= perf_trace_del,
11006	.start		= perf_swevent_start,
11007	.stop		= perf_swevent_stop,
11008	.read		= perf_swevent_read,
11009};
11010
11011static int perf_tp_filter_match(struct perf_event *event,
11012				struct perf_raw_record *raw)
11013{
11014	void *record = raw->frag.data;
11015
11016	/* only top level events have filters set */
11017	if (event->parent)
11018		event = event->parent;
11019
11020	if (likely(!event->filter) || filter_match_preds(event->filter, record))
11021		return 1;
11022	return 0;
11023}
11024
11025static int perf_tp_event_match(struct perf_event *event,
11026				struct perf_raw_record *raw,
11027				struct pt_regs *regs)
11028{
11029	if (event->hw.state & PERF_HES_STOPPED)
11030		return 0;
11031	/*
11032	 * If exclude_kernel, only trace user-space tracepoints (uprobes)
11033	 */
11034	if (event->attr.exclude_kernel && !user_mode(regs))
11035		return 0;
11036
11037	if (!perf_tp_filter_match(event, raw))
11038		return 0;
11039
11040	return 1;
11041}
11042
11043void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,
11044			       struct trace_event_call *call, u64 count,
11045			       struct pt_regs *regs, struct hlist_head *head,
11046			       struct task_struct *task)
11047{
11048	if (bpf_prog_array_valid(call)) {
11049		*(struct pt_regs **)raw_data = regs;
11050		if (!trace_call_bpf(call, raw_data) || hlist_empty(head)) {
11051			perf_swevent_put_recursion_context(rctx);
11052			return;
11053		}
11054	}
11055	perf_tp_event(call->event.type, count, raw_data, size, regs, head,
11056		      rctx, task);
11057}
11058EXPORT_SYMBOL_GPL(perf_trace_run_bpf_submit);
11059
11060static void __perf_tp_event_target_task(u64 count, void *record,
11061					struct pt_regs *regs,
11062					struct perf_sample_data *data,
11063					struct perf_raw_record *raw,
11064					struct perf_event *event)
11065{
11066	struct trace_entry *entry = record;
11067
11068	if (event->attr.config != entry->type)
11069		return;
11070	/* Cannot deliver synchronous signal to other task. */
11071	if (event->attr.sigtrap)
11072		return;
11073	if (perf_tp_event_match(event, raw, regs)) {
11074		perf_sample_data_init(data, 0, 0);
11075		perf_sample_save_raw_data(data, event, raw);
11076		perf_swevent_event(event, count, data, regs);
11077	}
11078}
11079
11080static void perf_tp_event_target_task(u64 count, void *record,
11081				      struct pt_regs *regs,
11082				      struct perf_sample_data *data,
11083				      struct perf_raw_record *raw,
11084				      struct perf_event_context *ctx)
11085{
11086	unsigned int cpu = smp_processor_id();
11087	struct pmu *pmu = &perf_tracepoint;
11088	struct perf_event *event, *sibling;
11089
11090	perf_event_groups_for_cpu_pmu(event, &ctx->pinned_groups, cpu, pmu) {
11091		__perf_tp_event_target_task(count, record, regs, data, raw, event);
11092		for_each_sibling_event(sibling, event)
11093			__perf_tp_event_target_task(count, record, regs, data, raw, sibling);
11094	}
11095
11096	perf_event_groups_for_cpu_pmu(event, &ctx->flexible_groups, cpu, pmu) {
11097		__perf_tp_event_target_task(count, record, regs, data, raw, event);
11098		for_each_sibling_event(sibling, event)
11099			__perf_tp_event_target_task(count, record, regs, data, raw, sibling);
11100	}
11101}
11102
11103void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size,
11104		   struct pt_regs *regs, struct hlist_head *head, int rctx,
11105		   struct task_struct *task)
11106{
11107	struct perf_sample_data data;
11108	struct perf_event *event;
11109
11110	/*
11111	 * Per being a tracepoint, this runs with preemption disabled.
11112	 */
11113	lockdep_assert_preemption_disabled();
11114
11115	struct perf_raw_record raw = {
11116		.frag = {
11117			.size = entry_size,
11118			.data = record,
11119		},
11120	};
11121
11122	perf_trace_buf_update(record, event_type);
11123
11124	hlist_for_each_entry_rcu(event, head, hlist_entry) {
11125		if (perf_tp_event_match(event, &raw, regs)) {
11126			/*
11127			 * Here use the same on-stack perf_sample_data,
11128			 * some members in data are event-specific and
11129			 * need to be re-computed for different sweveents.
11130			 * Re-initialize data->sample_flags safely to avoid
11131			 * the problem that next event skips preparing data
11132			 * because data->sample_flags is set.
11133			 */
11134			perf_sample_data_init(&data, 0, 0);
11135			perf_sample_save_raw_data(&data, event, &raw);
11136			perf_swevent_event(event, count, &data, regs);
11137		}
11138	}
11139
11140	/*
11141	 * If we got specified a target task, also iterate its context and
11142	 * deliver this event there too.
11143	 */
11144	if (task && task != current) {
11145		struct perf_event_context *ctx;
11146
11147		rcu_read_lock();
11148		ctx = rcu_dereference(task->perf_event_ctxp);
11149		if (!ctx)
11150			goto unlock;
11151
11152		raw_spin_lock(&ctx->lock);
11153		perf_tp_event_target_task(count, record, regs, &data, &raw, ctx);
11154		raw_spin_unlock(&ctx->lock);
11155unlock:
11156		rcu_read_unlock();
11157	}
11158
11159	perf_swevent_put_recursion_context(rctx);
11160}
11161EXPORT_SYMBOL_GPL(perf_tp_event);
11162
11163#if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
11164/*
11165 * Flags in config, used by dynamic PMU kprobe and uprobe
11166 * The flags should match following PMU_FORMAT_ATTR().
11167 *
11168 * PERF_PROBE_CONFIG_IS_RETPROBE if set, create kretprobe/uretprobe
11169 *                               if not set, create kprobe/uprobe
11170 *
11171 * The following values specify a reference counter (or semaphore in the
11172 * terminology of tools like dtrace, systemtap, etc.) Userspace Statically
11173 * Defined Tracepoints (USDT). Currently, we use 40 bit for the offset.
11174 *
11175 * PERF_UPROBE_REF_CTR_OFFSET_BITS	# of bits in config as th offset
11176 * PERF_UPROBE_REF_CTR_OFFSET_SHIFT	# of bits to shift left
11177 */
11178enum perf_probe_config {
11179	PERF_PROBE_CONFIG_IS_RETPROBE = 1U << 0,  /* [k,u]retprobe */
11180	PERF_UPROBE_REF_CTR_OFFSET_BITS = 32,
11181	PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS,
11182};
11183
11184PMU_FORMAT_ATTR(retprobe, "config:0");
11185#endif
11186
11187#ifdef CONFIG_KPROBE_EVENTS
11188static struct attribute *kprobe_attrs[] = {
11189	&format_attr_retprobe.attr,
11190	NULL,
11191};
11192
11193static struct attribute_group kprobe_format_group = {
11194	.name = "format",
11195	.attrs = kprobe_attrs,
11196};
11197
11198static const struct attribute_group *kprobe_attr_groups[] = {
11199	&kprobe_format_group,
11200	NULL,
11201};
11202
11203static int perf_kprobe_event_init(struct perf_event *event);
11204static struct pmu perf_kprobe = {
11205	.task_ctx_nr	= perf_sw_context,
11206	.event_init	= perf_kprobe_event_init,
11207	.add		= perf_trace_add,
11208	.del		= perf_trace_del,
11209	.start		= perf_swevent_start,
11210	.stop		= perf_swevent_stop,
11211	.read		= perf_swevent_read,
11212	.attr_groups	= kprobe_attr_groups,
11213};
11214
11215static int perf_kprobe_event_init(struct perf_event *event)
11216{
11217	int err;
11218	bool is_retprobe;
11219
11220	if (event->attr.type != perf_kprobe.type)
11221		return -ENOENT;
11222
11223	if (!perfmon_capable())
11224		return -EACCES;
11225
11226	/*
11227	 * no branch sampling for probe events
11228	 */
11229	if (has_branch_stack(event))
11230		return -EOPNOTSUPP;
11231
11232	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;
11233	err = perf_kprobe_init(event, is_retprobe);
11234	if (err)
11235		return err;
11236
11237	event->destroy = perf_kprobe_destroy;
11238
11239	return 0;
11240}
11241#endif /* CONFIG_KPROBE_EVENTS */
11242
11243#ifdef CONFIG_UPROBE_EVENTS
11244PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63");
11245
11246static struct attribute *uprobe_attrs[] = {
11247	&format_attr_retprobe.attr,
11248	&format_attr_ref_ctr_offset.attr,
11249	NULL,
11250};
11251
11252static struct attribute_group uprobe_format_group = {
11253	.name = "format",
11254	.attrs = uprobe_attrs,
11255};
11256
11257static const struct attribute_group *uprobe_attr_groups[] = {
11258	&uprobe_format_group,
11259	NULL,
11260};
11261
11262static int perf_uprobe_event_init(struct perf_event *event);
11263static struct pmu perf_uprobe = {
11264	.task_ctx_nr	= perf_sw_context,
11265	.event_init	= perf_uprobe_event_init,
11266	.add		= perf_trace_add,
11267	.del		= perf_trace_del,
11268	.start		= perf_swevent_start,
11269	.stop		= perf_swevent_stop,
11270	.read		= perf_swevent_read,
11271	.attr_groups	= uprobe_attr_groups,
11272};
11273
11274static int perf_uprobe_event_init(struct perf_event *event)
11275{
11276	int err;
11277	unsigned long ref_ctr_offset;
11278	bool is_retprobe;
11279
11280	if (event->attr.type != perf_uprobe.type)
11281		return -ENOENT;
11282
11283	if (!capable(CAP_SYS_ADMIN))
11284		return -EACCES;
11285
11286	/*
11287	 * no branch sampling for probe events
11288	 */
11289	if (has_branch_stack(event))
11290		return -EOPNOTSUPP;
11291
11292	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;
11293	ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
11294	err = perf_uprobe_init(event, ref_ctr_offset, is_retprobe);
11295	if (err)
11296		return err;
11297
11298	event->destroy = perf_uprobe_destroy;
11299
11300	return 0;
11301}
11302#endif /* CONFIG_UPROBE_EVENTS */
11303
11304static inline void perf_tp_register(void)
11305{
11306	perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
11307#ifdef CONFIG_KPROBE_EVENTS
11308	perf_pmu_register(&perf_kprobe, "kprobe", -1);
11309#endif
11310#ifdef CONFIG_UPROBE_EVENTS
11311	perf_pmu_register(&perf_uprobe, "uprobe", -1);
11312#endif
11313}
11314
11315static void perf_event_free_filter(struct perf_event *event)
11316{
11317	ftrace_profile_free_filter(event);
11318}
11319
11320/*
11321 * returns true if the event is a tracepoint, or a kprobe/upprobe created
11322 * with perf_event_open()
11323 */
11324static inline bool perf_event_is_tracing(struct perf_event *event)
11325{
11326	if (event->pmu == &perf_tracepoint)
11327		return true;
11328#ifdef CONFIG_KPROBE_EVENTS
11329	if (event->pmu == &perf_kprobe)
11330		return true;
11331#endif
11332#ifdef CONFIG_UPROBE_EVENTS
11333	if (event->pmu == &perf_uprobe)
11334		return true;
11335#endif
11336	return false;
11337}
11338
11339static int __perf_event_set_bpf_prog(struct perf_event *event,
11340				     struct bpf_prog *prog,
11341				     u64 bpf_cookie)
11342{
11343	bool is_kprobe, is_uprobe, is_tracepoint, is_syscall_tp;
11344
11345	if (event->state <= PERF_EVENT_STATE_REVOKED)
11346		return -ENODEV;
11347
11348	if (!perf_event_is_tracing(event))
11349		return perf_event_set_bpf_handler(event, prog, bpf_cookie);
11350
11351	is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_KPROBE;
11352	is_uprobe = event->tp_event->flags & TRACE_EVENT_FL_UPROBE;
11353	is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;
11354	is_syscall_tp = is_syscall_trace_event(event->tp_event);
11355	if (!is_kprobe && !is_uprobe && !is_tracepoint && !is_syscall_tp)
11356		/* bpf programs can only be attached to u/kprobe or tracepoint */
11357		return -EINVAL;
11358
11359	if (((is_kprobe || is_uprobe) && prog->type != BPF_PROG_TYPE_KPROBE) ||
11360	    (is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) ||
11361	    (is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT))
11362		return -EINVAL;
11363
11364	if (prog->type == BPF_PROG_TYPE_KPROBE && prog->sleepable && !is_uprobe)
11365		/* only uprobe programs are allowed to be sleepable */
11366		return -EINVAL;
11367
11368	/* Kprobe override only works for kprobes, not uprobes. */
11369	if (prog->kprobe_override && !is_kprobe)
11370		return -EINVAL;
11371
11372	/* Writing to context allowed only for uprobes. */
11373	if (prog->aux->kprobe_write_ctx && !is_uprobe)
11374		return -EINVAL;
11375
11376	if (is_tracepoint || is_syscall_tp) {
11377		int off = trace_event_get_offsets(event->tp_event);
11378
11379		if (prog->aux->max_ctx_offset > off)
11380			return -EACCES;
11381	}
11382
11383	return perf_event_attach_bpf_prog(event, prog, bpf_cookie);
11384}
11385
11386int perf_event_set_bpf_prog(struct perf_event *event,
11387			    struct bpf_prog *prog,
11388			    u64 bpf_cookie)
11389{
11390	struct perf_event_context *ctx;
11391	int ret;
11392
11393	ctx = perf_event_ctx_lock(event);
11394	ret = __perf_event_set_bpf_prog(event, prog, bpf_cookie);
11395	perf_event_ctx_unlock(event, ctx);
11396
11397	return ret;
11398}
11399
11400void perf_event_free_bpf_prog(struct perf_event *event)
11401{
11402	if (!event->prog)
11403		return;
11404
11405	if (!perf_event_is_tracing(event)) {
11406		perf_event_free_bpf_handler(event);
11407		return;
11408	}
11409	perf_event_detach_bpf_prog(event);
11410}
11411
11412#else
11413
11414static inline void perf_tp_register(void)
11415{
11416}
11417
11418static void perf_event_free_filter(struct perf_event *event)
11419{
11420}
11421
11422static int __perf_event_set_bpf_prog(struct perf_event *event,
11423				     struct bpf_prog *prog,
11424				     u64 bpf_cookie)
11425{
11426	return -ENOENT;
11427}
11428
11429int perf_event_set_bpf_prog(struct perf_event *event,
11430			    struct bpf_prog *prog,
11431			    u64 bpf_cookie)
11432{
11433	return -ENOENT;
11434}
11435
11436void perf_event_free_bpf_prog(struct perf_event *event)
11437{
11438}
11439#endif /* CONFIG_EVENT_TRACING */
11440
11441#ifdef CONFIG_HAVE_HW_BREAKPOINT
11442void perf_bp_event(struct perf_event *bp, void *data)
11443{
11444	struct perf_sample_data sample;
11445	struct pt_regs *regs = data;
11446
11447	/*
11448	 * Exception context, will have interrupts disabled.
11449	 */
11450	lockdep_assert_irqs_disabled();
11451
11452	perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
11453
11454	if (!bp->hw.state && !perf_exclude_event(bp, regs))
11455		perf_swevent_event(bp, 1, &sample, regs);
11456}
11457#endif
11458
11459/*
11460 * Allocate a new address filter
11461 */
11462static struct perf_addr_filter *
11463perf_addr_filter_new(struct perf_event *event, struct list_head *filters)
11464{
11465	int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu);
11466	struct perf_addr_filter *filter;
11467
11468	filter = kzalloc_node(sizeof(*filter), GFP_KERNEL, node);
11469	if (!filter)
11470		return NULL;
11471
11472	INIT_LIST_HEAD(&filter->entry);
11473	list_add_tail(&filter->entry, filters);
11474
11475	return filter;
11476}
11477
11478static void free_filters_list(struct list_head *filters)
11479{
11480	struct perf_addr_filter *filter, *iter;
11481
11482	list_for_each_entry_safe(filter, iter, filters, entry) {
11483		path_put(&filter->path);
11484		list_del(&filter->entry);
11485		kfree(filter);
11486	}
11487}
11488
11489/*
11490 * Free existing address filters and optionally install new ones
11491 */
11492static void perf_addr_filters_splice(struct perf_event *event,
11493				     struct list_head *head)
11494{
11495	unsigned long flags;
11496	LIST_HEAD(list);
11497
11498	if (!has_addr_filter(event))
11499		return;
11500
11501	/* don't bother with children, they don't have their own filters */
11502	if (event->parent)
11503		return;
11504
11505	raw_spin_lock_irqsave(&event->addr_filters.lock, flags);
11506
11507	list_splice_init(&event->addr_filters.list, &list);
11508	if (head)
11509		list_splice(head, &event->addr_filters.list);
11510
11511	raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags);
11512
11513	free_filters_list(&list);
11514}
11515
11516static void perf_free_addr_filters(struct perf_event *event)
11517{
11518	/*
11519	 * Used during free paths, there is no concurrency.
11520	 */
11521	if (list_empty(&event->addr_filters.list))
11522		return;
11523
11524	perf_addr_filters_splice(event, NULL);
11525}
11526
11527/*
11528 * Scan through mm's vmas and see if one of them matches the
11529 * @filter; if so, adjust filter's address range.
11530 * Called with mm::mmap_lock down for reading.
11531 */
11532static void perf_addr_filter_apply(struct perf_addr_filter *filter,
11533				   struct mm_struct *mm,
11534				   struct perf_addr_filter_range *fr)
11535{
11536	struct vm_area_struct *vma;
11537	VMA_ITERATOR(vmi, mm, 0);
11538
11539	for_each_vma(vmi, vma) {
11540		if (!vma->vm_file)
11541			continue;
11542
11543		if (perf_addr_filter_vma_adjust(filter, vma, fr))
11544			return;
11545	}
11546}
11547
11548/*
11549 * Update event's address range filters based on the
11550 * task's existing mappings, if any.
11551 */
11552static void perf_event_addr_filters_apply(struct perf_event *event)
11553{
11554	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
11555	struct task_struct *task = READ_ONCE(event->ctx->task);
11556	struct perf_addr_filter *filter;
11557	struct mm_struct *mm = NULL;
11558	unsigned int count = 0;
11559	unsigned long flags;
11560
11561	/*
11562	 * We may observe TASK_TOMBSTONE, which means that the event tear-down
11563	 * will stop on the parent's child_mutex that our caller is also holding
11564	 */
11565	if (task == TASK_TOMBSTONE)
11566		return;
11567
11568	if (ifh->nr_file_filters) {
11569		mm = get_task_mm(task);
11570		if (!mm)
11571			goto restart;
11572
11573		mmap_read_lock(mm);
11574	}
11575
11576	raw_spin_lock_irqsave(&ifh->lock, flags);
11577	list_for_each_entry(filter, &ifh->list, entry) {
11578		if (filter->path.dentry) {
11579			/*
11580			 * Adjust base offset if the filter is associated to a
11581			 * binary that needs to be mapped:
11582			 */
11583			event->addr_filter_ranges[count].start = 0;
11584			event->addr_filter_ranges[count].size = 0;
11585
11586			perf_addr_filter_apply(filter, mm, &event->addr_filter_ranges[count]);
11587		} else {
11588			event->addr_filter_ranges[count].start = filter->offset;
11589			event->addr_filter_ranges[count].size  = filter->size;
11590		}
11591
11592		count++;
11593	}
11594
11595	event->addr_filters_gen++;
11596	raw_spin_unlock_irqrestore(&ifh->lock, flags);
11597
11598	if (ifh->nr_file_filters) {
11599		mmap_read_unlock(mm);
11600
11601		mmput(mm);
11602	}
11603
11604restart:
11605	perf_event_stop(event, 1);
11606}
11607
11608/*
11609 * Address range filtering: limiting the data to certain
11610 * instruction address ranges. Filters are ioctl()ed to us from
11611 * userspace as ascii strings.
11612 *
11613 * Filter string format:
11614 *
11615 * ACTION RANGE_SPEC
11616 * where ACTION is one of the
11617 *  * "filter": limit the trace to this region
11618 *  * "start": start tracing from this address
11619 *  * "stop": stop tracing at this address/region;
11620 * RANGE_SPEC is
11621 *  * for kernel addresses: <start address>[/<size>]
11622 *  * for object files:     <start address>[/<size>]@</path/to/object/file>
11623 *
11624 * if <size> is not specified or is zero, the range is treated as a single
11625 * address; not valid for ACTION=="filter".
11626 */
11627enum {
11628	IF_ACT_NONE = -1,
11629	IF_ACT_FILTER,
11630	IF_ACT_START,
11631	IF_ACT_STOP,
11632	IF_SRC_FILE,
11633	IF_SRC_KERNEL,
11634	IF_SRC_FILEADDR,
11635	IF_SRC_KERNELADDR,
11636};
11637
11638enum {
11639	IF_STATE_ACTION = 0,
11640	IF_STATE_SOURCE,
11641	IF_STATE_END,
11642};
11643
11644static const match_table_t if_tokens = {
11645	{ IF_ACT_FILTER,	"filter" },
11646	{ IF_ACT_START,		"start" },
11647	{ IF_ACT_STOP,		"stop" },
11648	{ IF_SRC_FILE,		"%u/%u@%s" },
11649	{ IF_SRC_KERNEL,	"%u/%u" },
11650	{ IF_SRC_FILEADDR,	"%u@%s" },
11651	{ IF_SRC_KERNELADDR,	"%u" },
11652	{ IF_ACT_NONE,		NULL },
11653};
11654
11655/*
11656 * Address filter string parser
11657 */
11658static int
11659perf_event_parse_addr_filter(struct perf_event *event, char *fstr,
11660			     struct list_head *filters)
11661{
11662	struct perf_addr_filter *filter = NULL;
11663	char *start, *orig, *filename = NULL;
11664	substring_t args[MAX_OPT_ARGS];
11665	int state = IF_STATE_ACTION, token;
11666	unsigned int kernel = 0;
11667	int ret = -EINVAL;
11668
11669	orig = fstr = kstrdup(fstr, GFP_KERNEL);
11670	if (!fstr)
11671		return -ENOMEM;
11672
11673	while ((start = strsep(&fstr, " ,\n")) != NULL) {
11674		static const enum perf_addr_filter_action_t actions[] = {
11675			[IF_ACT_FILTER]	= PERF_ADDR_FILTER_ACTION_FILTER,
11676			[IF_ACT_START]	= PERF_ADDR_FILTER_ACTION_START,
11677			[IF_ACT_STOP]	= PERF_ADDR_FILTER_ACTION_STOP,
11678		};
11679		ret = -EINVAL;
11680
11681		if (!*start)
11682			continue;
11683
11684		/* filter definition begins */
11685		if (state == IF_STATE_ACTION) {
11686			filter = perf_addr_filter_new(event, filters);
11687			if (!filter)
11688				goto fail;
11689		}
11690
11691		token = match_token(start, if_tokens, args);
11692		switch (token) {
11693		case IF_ACT_FILTER:
11694		case IF_ACT_START:
11695		case IF_ACT_STOP:
11696			if (state != IF_STATE_ACTION)
11697				goto fail;
11698
11699			filter->action = actions[token];
11700			state = IF_STATE_SOURCE;
11701			break;
11702
11703		case IF_SRC_KERNELADDR:
11704		case IF_SRC_KERNEL:
11705			kernel = 1;
11706			fallthrough;
11707
11708		case IF_SRC_FILEADDR:
11709		case IF_SRC_FILE:
11710			if (state != IF_STATE_SOURCE)
11711				goto fail;
11712
11713			*args[0].to = 0;
11714			ret = kstrtoul(args[0].from, 0, &filter->offset);
11715			if (ret)
11716				goto fail;
11717
11718			if (token == IF_SRC_KERNEL || token == IF_SRC_FILE) {
11719				*args[1].to = 0;
11720				ret = kstrtoul(args[1].from, 0, &filter->size);
11721				if (ret)
11722					goto fail;
11723			}
11724
11725			if (token == IF_SRC_FILE || token == IF_SRC_FILEADDR) {
11726				int fpos = token == IF_SRC_FILE ? 2 : 1;
11727
11728				kfree(filename);
11729				filename = match_strdup(&args[fpos]);
11730				if (!filename) {
11731					ret = -ENOMEM;
11732					goto fail;
11733				}
11734			}
11735
11736			state = IF_STATE_END;
11737			break;
11738
11739		default:
11740			goto fail;
11741		}
11742
11743		/*
11744		 * Filter definition is fully parsed, validate and install it.
11745		 * Make sure that it doesn't contradict itself or the event's
11746		 * attribute.
11747		 */
11748		if (state == IF_STATE_END) {
11749			ret = -EINVAL;
11750
11751			/*
11752			 * ACTION "filter" must have a non-zero length region
11753			 * specified.
11754			 */
11755			if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER &&
11756			    !filter->size)
11757				goto fail;
11758
11759			if (!kernel) {
11760				if (!filename)
11761					goto fail;
11762
11763				/*
11764				 * For now, we only support file-based filters
11765				 * in per-task events; doing so for CPU-wide
11766				 * events requires additional context switching
11767				 * trickery, since same object code will be
11768				 * mapped at different virtual addresses in
11769				 * different processes.
11770				 */
11771				ret = -EOPNOTSUPP;
11772				if (!event->ctx->task)
11773					goto fail;
11774
11775				/* look up the path and grab its inode */
11776				ret = kern_path(filename, LOOKUP_FOLLOW,
11777						&filter->path);
11778				if (ret)
11779					goto fail;
11780
11781				ret = -EINVAL;
11782				if (!filter->path.dentry ||
11783				    !S_ISREG(d_inode(filter->path.dentry)
11784					     ->i_mode))
11785					goto fail;
11786
11787				event->addr_filters.nr_file_filters++;
11788			}
11789
11790			/* ready to consume more filters */
11791			kfree(filename);
11792			filename = NULL;
11793			state = IF_STATE_ACTION;
11794			filter = NULL;
11795			kernel = 0;
11796		}
11797	}
11798
11799	if (state != IF_STATE_ACTION)
11800		goto fail;
11801
11802	kfree(filename);
11803	kfree(orig);
11804
11805	return 0;
11806
11807fail:
11808	kfree(filename);
11809	free_filters_list(filters);
11810	kfree(orig);
11811
11812	return ret;
11813}
11814
11815static int
11816perf_event_set_addr_filter(struct perf_event *event, char *filter_str)
11817{
11818	LIST_HEAD(filters);
11819	int ret;
11820
11821	/*
11822	 * Since this is called in perf_ioctl() path, we're already holding
11823	 * ctx::mutex.
11824	 */
11825	lockdep_assert_held(&event->ctx->mutex);
11826
11827	if (WARN_ON_ONCE(event->parent))
11828		return -EINVAL;
11829
11830	ret = perf_event_parse_addr_filter(event, filter_str, &filters);
11831	if (ret)
11832		goto fail_clear_files;
11833
11834	ret = event->pmu->addr_filters_validate(&filters);
11835	if (ret)
11836		goto fail_free_filters;
11837
11838	/* remove existing filters, if any */
11839	perf_addr_filters_splice(event, &filters);
11840
11841	/* install new filters */
11842	perf_event_for_each_child(event, perf_event_addr_filters_apply);
11843
11844	return ret;
11845
11846fail_free_filters:
11847	free_filters_list(&filters);
11848
11849fail_clear_files:
11850	event->addr_filters.nr_file_filters = 0;
11851
11852	return ret;
11853}
11854
11855static int perf_event_set_filter(struct perf_event *event, void __user *arg)
11856{
11857	int ret = -EINVAL;
11858	char *filter_str;
11859
11860	filter_str = strndup_user(arg, PAGE_SIZE);
11861	if (IS_ERR(filter_str))
11862		return PTR_ERR(filter_str);
11863
11864#ifdef CONFIG_EVENT_TRACING
11865	if (perf_event_is_tracing(event)) {
11866		struct perf_event_context *ctx = event->ctx;
11867
11868		/*
11869		 * Beware, here be dragons!!
11870		 *
11871		 * the tracepoint muck will deadlock against ctx->mutex, but
11872		 * the tracepoint stuff does not actually need it. So
11873		 * temporarily drop ctx->mutex. As per perf_event_ctx_lock() we
11874		 * already have a reference on ctx.
11875		 *
11876		 * This can result in event getting moved to a different ctx,
11877		 * but that does not affect the tracepoint state.
11878		 */
11879		mutex_unlock(&ctx->mutex);
11880		ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
11881		mutex_lock(&ctx->mutex);
11882	} else
11883#endif
11884	if (has_addr_filter(event))
11885		ret = perf_event_set_addr_filter(event, filter_str);
11886
11887	kfree(filter_str);
11888	return ret;
11889}
11890
11891/*
11892 * hrtimer based swevent callback
11893 */
11894
11895static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
11896{
11897	enum hrtimer_restart ret = HRTIMER_RESTART;
11898	struct perf_sample_data data;
11899	struct pt_regs *regs;
11900	struct perf_event *event;
11901	u64 period;
11902
11903	event = container_of(hrtimer, struct perf_event, hw.hrtimer);
11904
11905	if (event->state != PERF_EVENT_STATE_ACTIVE ||
11906	    event->hw.state & PERF_HES_STOPPED)
11907		return HRTIMER_NORESTART;
11908
11909	event->pmu->read(event);
11910
11911	perf_sample_data_init(&data, 0, event->hw.last_period);
11912	regs = get_irq_regs();
11913
11914	if (regs && !perf_exclude_event(event, regs)) {
11915		if (!(event->attr.exclude_idle && is_idle_task(current)))
11916			if (perf_event_overflow(event, &data, regs))
11917				ret = HRTIMER_NORESTART;
11918	}
11919
11920	period = max_t(u64, 10000, event->hw.sample_period);
11921	hrtimer_forward_now(hrtimer, ns_to_ktime(period));
11922
11923	return ret;
11924}
11925
11926static void perf_swevent_start_hrtimer(struct perf_event *event)
11927{
11928	struct hw_perf_event *hwc = &event->hw;
11929	s64 period;
11930
11931	if (!is_sampling_event(event))
11932		return;
11933
11934	period = local64_read(&hwc->period_left);
11935	if (period) {
11936		if (period < 0)
11937			period = 10000;
11938
11939		local64_set(&hwc->period_left, 0);
11940	} else {
11941		period = max_t(u64, 10000, hwc->sample_period);
11942	}
11943	hrtimer_start(&hwc->hrtimer, ns_to_ktime(period),
11944		      HRTIMER_MODE_REL_PINNED_HARD);
11945}
11946
11947static void perf_swevent_cancel_hrtimer(struct perf_event *event)
11948{
11949	struct hw_perf_event *hwc = &event->hw;
11950
11951	/*
11952	 * Careful: this function can be triggered in the hrtimer handler,
11953	 * for cpu-clock events, so hrtimer_cancel() would cause a
11954	 * deadlock.
11955	 *
11956	 * So use hrtimer_try_to_cancel() to try to stop the hrtimer,
11957	 * and the cpu-clock handler also sets the PERF_HES_STOPPED flag,
11958	 * which guarantees that perf_swevent_hrtimer() will stop the
11959	 * hrtimer once it sees the PERF_HES_STOPPED flag.
11960	 */
11961	if (is_sampling_event(event) && (hwc->interrupts != MAX_INTERRUPTS)) {
11962		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
11963		local64_set(&hwc->period_left, ktime_to_ns(remaining));
11964
11965		hrtimer_try_to_cancel(&hwc->hrtimer);
11966	}
11967}
11968
11969static void perf_swevent_destroy_hrtimer(struct perf_event *event)
11970{
11971	hrtimer_cancel(&event->hw.hrtimer);
11972}
11973
11974static void perf_swevent_init_hrtimer(struct perf_event *event)
11975{
11976	struct hw_perf_event *hwc = &event->hw;
11977
11978	if (!is_sampling_event(event))
11979		return;
11980
11981	hrtimer_setup(&hwc->hrtimer, perf_swevent_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
11982	event->destroy = perf_swevent_destroy_hrtimer;
11983
11984	/*
11985	 * Since hrtimers have a fixed rate, we can do a static freq->period
11986	 * mapping and avoid the whole period adjust feedback stuff.
11987	 */
11988	if (event->attr.freq) {
11989		long freq = event->attr.sample_freq;
11990
11991		event->attr.sample_period = NSEC_PER_SEC / freq;
11992		hwc->sample_period = event->attr.sample_period;
11993		local64_set(&hwc->period_left, hwc->sample_period);
11994		hwc->last_period = hwc->sample_period;
11995		event->attr.freq = 0;
11996	}
11997}
11998
11999/*
12000 * Software event: cpu wall time clock
12001 */
12002
12003static void cpu_clock_event_update(struct perf_event *event)
12004{
12005	s64 prev;
12006	u64 now;
12007
12008	now = local_clock();
12009	prev = local64_xchg(&event->hw.prev_count, now);
12010	local64_add(now - prev, &event->count);
12011}
12012
12013static void cpu_clock_event_start(struct perf_event *event, int flags)
12014{
12015	event->hw.state = 0;
12016	local64_set(&event->hw.prev_count, local_clock());
12017	perf_swevent_start_hrtimer(event);
12018}
12019
12020static void cpu_clock_event_stop(struct perf_event *event, int flags)
12021{
12022	event->hw.state = PERF_HES_STOPPED;
12023	perf_swevent_cancel_hrtimer(event);
12024	if (flags & PERF_EF_UPDATE)
12025		cpu_clock_event_update(event);
12026}
12027
12028static int cpu_clock_event_add(struct perf_event *event, int flags)
12029{
12030	if (flags & PERF_EF_START)
12031		cpu_clock_event_start(event, flags);
12032	perf_event_update_userpage(event);
12033
12034	return 0;
12035}
12036
12037static void cpu_clock_event_del(struct perf_event *event, int flags)
12038{
12039	cpu_clock_event_stop(event, PERF_EF_UPDATE);
12040}
12041
12042static void cpu_clock_event_read(struct perf_event *event)
12043{
12044	cpu_clock_event_update(event);
12045}
12046
12047static int cpu_clock_event_init(struct perf_event *event)
12048{
12049	if (event->attr.type != perf_cpu_clock.type)
12050		return -ENOENT;
12051
12052	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
12053		return -ENOENT;
12054
12055	/*
12056	 * no branch sampling for software events
12057	 */
12058	if (has_branch_stack(event))
12059		return -EOPNOTSUPP;
12060
12061	perf_swevent_init_hrtimer(event);
12062
12063	return 0;
12064}
12065
12066static struct pmu perf_cpu_clock = {
12067	.task_ctx_nr	= perf_sw_context,
12068
12069	.capabilities	= PERF_PMU_CAP_NO_NMI,
12070	.dev		= PMU_NULL_DEV,
12071
12072	.event_init	= cpu_clock_event_init,
12073	.add		= cpu_clock_event_add,
12074	.del		= cpu_clock_event_del,
12075	.start		= cpu_clock_event_start,
12076	.stop		= cpu_clock_event_stop,
12077	.read		= cpu_clock_event_read,
12078};
12079
12080/*
12081 * Software event: task time clock
12082 */
12083
12084static void task_clock_event_update(struct perf_event *event, u64 now)
12085{
12086	u64 prev;
12087	s64 delta;
12088
12089	prev = local64_xchg(&event->hw.prev_count, now);
12090	delta = now - prev;
12091	local64_add(delta, &event->count);
12092}
12093
12094static void task_clock_event_start(struct perf_event *event, int flags)
12095{
12096	event->hw.state = 0;
12097	local64_set(&event->hw.prev_count, event->ctx->time);
12098	perf_swevent_start_hrtimer(event);
12099}
12100
12101static void task_clock_event_stop(struct perf_event *event, int flags)
12102{
12103	event->hw.state = PERF_HES_STOPPED;
12104	perf_swevent_cancel_hrtimer(event);
12105	if (flags & PERF_EF_UPDATE)
12106		task_clock_event_update(event, event->ctx->time);
12107}
12108
12109static int task_clock_event_add(struct perf_event *event, int flags)
12110{
12111	if (flags & PERF_EF_START)
12112		task_clock_event_start(event, flags);
12113	perf_event_update_userpage(event);
12114
12115	return 0;
12116}
12117
12118static void task_clock_event_del(struct perf_event *event, int flags)
12119{
12120	task_clock_event_stop(event, PERF_EF_UPDATE);
12121}
12122
12123static void task_clock_event_read(struct perf_event *event)
12124{
12125	u64 now = perf_clock();
12126	u64 delta = now - event->ctx->timestamp;
12127	u64 time = event->ctx->time + delta;
12128
12129	task_clock_event_update(event, time);
12130}
12131
12132static int task_clock_event_init(struct perf_event *event)
12133{
12134	if (event->attr.type != perf_task_clock.type)
12135		return -ENOENT;
12136
12137	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
12138		return -ENOENT;
12139
12140	/*
12141	 * no branch sampling for software events
12142	 */
12143	if (has_branch_stack(event))
12144		return -EOPNOTSUPP;
12145
12146	perf_swevent_init_hrtimer(event);
12147
12148	return 0;
12149}
12150
12151static struct pmu perf_task_clock = {
12152	.task_ctx_nr	= perf_sw_context,
12153
12154	.capabilities	= PERF_PMU_CAP_NO_NMI,
12155	.dev		= PMU_NULL_DEV,
12156
12157	.event_init	= task_clock_event_init,
12158	.add		= task_clock_event_add,
12159	.del		= task_clock_event_del,
12160	.start		= task_clock_event_start,
12161	.stop		= task_clock_event_stop,
12162	.read		= task_clock_event_read,
12163};
12164
12165static void perf_pmu_nop_void(struct pmu *pmu)
12166{
12167}
12168
12169static void perf_pmu_nop_txn(struct pmu *pmu, unsigned int flags)
12170{
12171}
12172
12173static int perf_pmu_nop_int(struct pmu *pmu)
12174{
12175	return 0;
12176}
12177
12178static int perf_event_nop_int(struct perf_event *event, u64 value)
12179{
12180	return 0;
12181}
12182
12183static DEFINE_PER_CPU(unsigned int, nop_txn_flags);
12184
12185static void perf_pmu_start_txn(struct pmu *pmu, unsigned int flags)
12186{
12187	__this_cpu_write(nop_txn_flags, flags);
12188
12189	if (flags & ~PERF_PMU_TXN_ADD)
12190		return;
12191
12192	perf_pmu_disable(pmu);
12193}
12194
12195static int perf_pmu_commit_txn(struct pmu *pmu)
12196{
12197	unsigned int flags = __this_cpu_read(nop_txn_flags);
12198
12199	__this_cpu_write(nop_txn_flags, 0);
12200
12201	if (flags & ~PERF_PMU_TXN_ADD)
12202		return 0;
12203
12204	perf_pmu_enable(pmu);
12205	return 0;
12206}
12207
12208static void perf_pmu_cancel_txn(struct pmu *pmu)
12209{
12210	unsigned int flags =  __this_cpu_read(nop_txn_flags);
12211
12212	__this_cpu_write(nop_txn_flags, 0);
12213
12214	if (flags & ~PERF_PMU_TXN_ADD)
12215		return;
12216
12217	perf_pmu_enable(pmu);
12218}
12219
12220static int perf_event_idx_default(struct perf_event *event)
12221{
12222	return 0;
12223}
12224
12225/*
12226 * Let userspace know that this PMU supports address range filtering:
12227 */
12228static ssize_t nr_addr_filters_show(struct device *dev,
12229				    struct device_attribute *attr,
12230				    char *page)
12231{
12232	struct pmu *pmu = dev_get_drvdata(dev);
12233
12234	return sysfs_emit(page, "%d\n", pmu->nr_addr_filters);
12235}
12236DEVICE_ATTR_RO(nr_addr_filters);
12237
12238static struct idr pmu_idr;
12239
12240static ssize_t
12241type_show(struct device *dev, struct device_attribute *attr, char *page)
12242{
12243	struct pmu *pmu = dev_get_drvdata(dev);
12244
12245	return sysfs_emit(page, "%d\n", pmu->type);
12246}
12247static DEVICE_ATTR_RO(type);
12248
12249static ssize_t
12250perf_event_mux_interval_ms_show(struct device *dev,
12251				struct device_attribute *attr,
12252				char *page)
12253{
12254	struct pmu *pmu = dev_get_drvdata(dev);
12255
12256	return sysfs_emit(page, "%d\n", pmu->hrtimer_interval_ms);
12257}
12258
12259static DEFINE_MUTEX(mux_interval_mutex);
12260
12261static ssize_t
12262perf_event_mux_interval_ms_store(struct device *dev,
12263				 struct device_attribute *attr,
12264				 const char *buf, size_t count)
12265{
12266	struct pmu *pmu = dev_get_drvdata(dev);
12267	int timer, cpu, ret;
12268
12269	ret = kstrtoint(buf, 0, &timer);
12270	if (ret)
12271		return ret;
12272
12273	if (timer < 1)
12274		return -EINVAL;
12275
12276	/* same value, noting to do */
12277	if (timer == pmu->hrtimer_interval_ms)
12278		return count;
12279
12280	mutex_lock(&mux_interval_mutex);
12281	pmu->hrtimer_interval_ms = timer;
12282
12283	/* update all cpuctx for this PMU */
12284	cpus_read_lock();
12285	for_each_online_cpu(cpu) {
12286		struct perf_cpu_pmu_context *cpc;
12287		cpc = *per_cpu_ptr(pmu->cpu_pmu_context, cpu);
12288		cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
12289
12290		cpu_function_call(cpu, perf_mux_hrtimer_restart_ipi, cpc);
12291	}
12292	cpus_read_unlock();
12293	mutex_unlock(&mux_interval_mutex);
12294
12295	return count;
12296}
12297static DEVICE_ATTR_RW(perf_event_mux_interval_ms);
12298
12299static inline const struct cpumask *perf_scope_cpu_topology_cpumask(unsigned int scope, int cpu)
12300{
12301	switch (scope) {
12302	case PERF_PMU_SCOPE_CORE:
12303		return topology_sibling_cpumask(cpu);
12304	case PERF_PMU_SCOPE_DIE:
12305		return topology_die_cpumask(cpu);
12306	case PERF_PMU_SCOPE_CLUSTER:
12307		return topology_cluster_cpumask(cpu);
12308	case PERF_PMU_SCOPE_PKG:
12309		return topology_core_cpumask(cpu);
12310	case PERF_PMU_SCOPE_SYS_WIDE:
12311		return cpu_online_mask;
12312	}
12313
12314	return NULL;
12315}
12316
12317static inline struct cpumask *perf_scope_cpumask(unsigned int scope)
12318{
12319	switch (scope) {
12320	case PERF_PMU_SCOPE_CORE:
12321		return perf_online_core_mask;
12322	case PERF_PMU_SCOPE_DIE:
12323		return perf_online_die_mask;
12324	case PERF_PMU_SCOPE_CLUSTER:
12325		return perf_online_cluster_mask;
12326	case PERF_PMU_SCOPE_PKG:
12327		return perf_online_pkg_mask;
12328	case PERF_PMU_SCOPE_SYS_WIDE:
12329		return perf_online_sys_mask;
12330	}
12331
12332	return NULL;
12333}
12334
12335static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
12336			    char *buf)
12337{
12338	struct pmu *pmu = dev_get_drvdata(dev);
12339	struct cpumask *mask = perf_scope_cpumask(pmu->scope);
12340
12341	if (mask)
12342		return cpumap_print_to_pagebuf(true, buf, mask);
12343	return 0;
12344}
12345
12346static DEVICE_ATTR_RO(cpumask);
12347
12348static struct attribute *pmu_dev_attrs[] = {
12349	&dev_attr_type.attr,
12350	&dev_attr_perf_event_mux_interval_ms.attr,
12351	&dev_attr_nr_addr_filters.attr,
12352	&dev_attr_cpumask.attr,
12353	NULL,
12354};
12355
12356static umode_t pmu_dev_is_visible(struct kobject *kobj, struct attribute *a, int n)
12357{
12358	struct device *dev = kobj_to_dev(kobj);
12359	struct pmu *pmu = dev_get_drvdata(dev);
12360
12361	if (n == 2 && !pmu->nr_addr_filters)
12362		return 0;
12363
12364	/* cpumask */
12365	if (n == 3 && pmu->scope == PERF_PMU_SCOPE_NONE)
12366		return 0;
12367
12368	return a->mode;
12369}
12370
12371static struct attribute_group pmu_dev_attr_group = {
12372	.is_visible = pmu_dev_is_visible,
12373	.attrs = pmu_dev_attrs,
12374};
12375
12376static const struct attribute_group *pmu_dev_groups[] = {
12377	&pmu_dev_attr_group,
12378	NULL,
12379};
12380
12381static int pmu_bus_running;
12382static const struct bus_type pmu_bus = {
12383	.name		= "event_source",
12384	.dev_groups	= pmu_dev_groups,
12385};
12386
12387static void pmu_dev_release(struct device *dev)
12388{
12389	kfree(dev);
12390}
12391
12392static int pmu_dev_alloc(struct pmu *pmu)
12393{
12394	int ret = -ENOMEM;
12395
12396	pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
12397	if (!pmu->dev)
12398		goto out;
12399
12400	pmu->dev->groups = pmu->attr_groups;
12401	device_initialize(pmu->dev);
12402
12403	dev_set_drvdata(pmu->dev, pmu);
12404	pmu->dev->bus = &pmu_bus;
12405	pmu->dev->parent = pmu->parent;
12406	pmu->dev->release = pmu_dev_release;
12407
12408	ret = dev_set_name(pmu->dev, "%s", pmu->name);
12409	if (ret)
12410		goto free_dev;
12411
12412	ret = device_add(pmu->dev);
12413	if (ret)
12414		goto free_dev;
12415
12416	if (pmu->attr_update) {
12417		ret = sysfs_update_groups(&pmu->dev->kobj, pmu->attr_update);
12418		if (ret)
12419			goto del_dev;
12420	}
12421
12422out:
12423	return ret;
12424
12425del_dev:
12426	device_del(pmu->dev);
12427
12428free_dev:
12429	put_device(pmu->dev);
12430	pmu->dev = NULL;
12431	goto out;
12432}
12433
12434static struct lock_class_key cpuctx_mutex;
12435static struct lock_class_key cpuctx_lock;
12436
12437static bool idr_cmpxchg(struct idr *idr, unsigned long id, void *old, void *new)
12438{
12439	void *tmp, *val = idr_find(idr, id);
12440
12441	if (val != old)
12442		return false;
12443
12444	tmp = idr_replace(idr, new, id);
12445	if (IS_ERR(tmp))
12446		return false;
12447
12448	WARN_ON_ONCE(tmp != val);
12449	return true;
12450}
12451
12452static void perf_pmu_free(struct pmu *pmu)
12453{
12454	if (pmu_bus_running && pmu->dev && pmu->dev != PMU_NULL_DEV) {
12455		if (pmu->nr_addr_filters)
12456			device_remove_file(pmu->dev, &dev_attr_nr_addr_filters);
12457		device_del(pmu->dev);
12458		put_device(pmu->dev);
12459	}
12460
12461	if (pmu->cpu_pmu_context) {
12462		int cpu;
12463
12464		for_each_possible_cpu(cpu) {
12465			struct perf_cpu_pmu_context *cpc;
12466
12467			cpc = *per_cpu_ptr(pmu->cpu_pmu_context, cpu);
12468			if (!cpc)
12469				continue;
12470			if (cpc->epc.embedded) {
12471				/* refcount managed */
12472				put_pmu_ctx(&cpc->epc);
12473				continue;
12474			}
12475			kfree(cpc);
12476		}
12477		free_percpu(pmu->cpu_pmu_context);
12478	}
12479}
12480
12481DEFINE_FREE(pmu_unregister, struct pmu *, if (_T) perf_pmu_free(_T))
12482
12483int perf_pmu_register(struct pmu *_pmu, const char *name, int type)
12484{
12485	int cpu, max = PERF_TYPE_MAX;
12486
12487	struct pmu *pmu __free(pmu_unregister) = _pmu;
12488	guard(mutex)(&pmus_lock);
12489
12490	if (WARN_ONCE(!name, "Can not register anonymous pmu.\n"))
12491		return -EINVAL;
12492
12493	if (WARN_ONCE(pmu->scope >= PERF_PMU_MAX_SCOPE,
12494		      "Can not register a pmu with an invalid scope.\n"))
12495		return -EINVAL;
12496
12497	pmu->name = name;
12498
12499	if (type >= 0)
12500		max = type;
12501
12502	CLASS(idr_alloc, pmu_type)(&pmu_idr, NULL, max, 0, GFP_KERNEL);
12503	if (pmu_type.id < 0)
12504		return pmu_type.id;
12505
12506	WARN_ON(type >= 0 && pmu_type.id != type);
12507
12508	pmu->type = pmu_type.id;
12509	atomic_set(&pmu->exclusive_cnt, 0);
12510
12511	if (pmu_bus_running && !pmu->dev) {
12512		int ret = pmu_dev_alloc(pmu);
12513		if (ret)
12514			return ret;
12515	}
12516
12517	pmu->cpu_pmu_context = alloc_percpu(struct perf_cpu_pmu_context *);
12518	if (!pmu->cpu_pmu_context)
12519		return -ENOMEM;
12520
12521	for_each_possible_cpu(cpu) {
12522		struct perf_cpu_pmu_context *cpc =
12523			kmalloc_node(sizeof(struct perf_cpu_pmu_context),
12524				     GFP_KERNEL | __GFP_ZERO,
12525				     cpu_to_node(cpu));
12526
12527		if (!cpc)
12528			return -ENOMEM;
12529
12530		*per_cpu_ptr(pmu->cpu_pmu_context, cpu) = cpc;
12531		__perf_init_event_pmu_context(&cpc->epc, pmu);
12532		__perf_mux_hrtimer_init(cpc, cpu);
12533	}
12534
12535	if (!pmu->start_txn) {
12536		if (pmu->pmu_enable) {
12537			/*
12538			 * If we have pmu_enable/pmu_disable calls, install
12539			 * transaction stubs that use that to try and batch
12540			 * hardware accesses.
12541			 */
12542			pmu->start_txn  = perf_pmu_start_txn;
12543			pmu->commit_txn = perf_pmu_commit_txn;
12544			pmu->cancel_txn = perf_pmu_cancel_txn;
12545		} else {
12546			pmu->start_txn  = perf_pmu_nop_txn;
12547			pmu->commit_txn = perf_pmu_nop_int;
12548			pmu->cancel_txn = perf_pmu_nop_void;
12549		}
12550	}
12551
12552	if (!pmu->pmu_enable) {
12553		pmu->pmu_enable  = perf_pmu_nop_void;
12554		pmu->pmu_disable = perf_pmu_nop_void;
12555	}
12556
12557	if (!pmu->check_period)
12558		pmu->check_period = perf_event_nop_int;
12559
12560	if (!pmu->event_idx)
12561		pmu->event_idx = perf_event_idx_default;
12562
12563	INIT_LIST_HEAD(&pmu->events);
12564	spin_lock_init(&pmu->events_lock);
12565
12566	/*
12567	 * Now that the PMU is complete, make it visible to perf_try_init_event().
12568	 */
12569	if (!idr_cmpxchg(&pmu_idr, pmu->type, NULL, pmu))
12570		return -EINVAL;
12571	list_add_rcu(&pmu->entry, &pmus);
12572
12573	take_idr_id(pmu_type);
12574	_pmu = no_free_ptr(pmu); // let it rip
12575	return 0;
12576}
12577EXPORT_SYMBOL_GPL(perf_pmu_register);
12578
12579static void __pmu_detach_event(struct pmu *pmu, struct perf_event *event,
12580			       struct perf_event_context *ctx)
12581{
12582	/*
12583	 * De-schedule the event and mark it REVOKED.
12584	 */
12585	perf_event_exit_event(event, ctx, ctx->task, true);
12586
12587	/*
12588	 * All _free_event() bits that rely on event->pmu:
12589	 *
12590	 * Notably, perf_mmap() relies on the ordering here.
12591	 */
12592	scoped_guard (mutex, &event->mmap_mutex) {
12593		WARN_ON_ONCE(pmu->event_unmapped);
12594		/*
12595		 * Mostly an empty lock sequence, such that perf_mmap(), which
12596		 * relies on mmap_mutex, is sure to observe the state change.
12597		 */
12598	}
12599
12600	perf_event_free_bpf_prog(event);
12601	perf_free_addr_filters(event);
12602
12603	if (event->destroy) {
12604		event->destroy(event);
12605		event->destroy = NULL;
12606	}
12607
12608	if (event->pmu_ctx) {
12609		put_pmu_ctx(event->pmu_ctx);
12610		event->pmu_ctx = NULL;
12611	}
12612
12613	exclusive_event_destroy(event);
12614	module_put(pmu->module);
12615
12616	event->pmu = NULL; /* force fault instead of UAF */
12617}
12618
12619static void pmu_detach_event(struct pmu *pmu, struct perf_event *event)
12620{
12621	struct perf_event_context *ctx;
12622
12623	ctx = perf_event_ctx_lock(event);
12624	__pmu_detach_event(pmu, event, ctx);
12625	perf_event_ctx_unlock(event, ctx);
12626
12627	scoped_guard (spinlock, &pmu->events_lock)
12628		list_del(&event->pmu_list);
12629}
12630
12631static struct perf_event *pmu_get_event(struct pmu *pmu)
12632{
12633	struct perf_event *event;
12634
12635	guard(spinlock)(&pmu->events_lock);
12636	list_for_each_entry(event, &pmu->events, pmu_list) {
12637		if (atomic_long_inc_not_zero(&event->refcount))
12638			return event;
12639	}
12640
12641	return NULL;
12642}
12643
12644static bool pmu_empty(struct pmu *pmu)
12645{
12646	guard(spinlock)(&pmu->events_lock);
12647	return list_empty(&pmu->events);
12648}
12649
12650static void pmu_detach_events(struct pmu *pmu)
12651{
12652	struct perf_event *event;
12653
12654	for (;;) {
12655		event = pmu_get_event(pmu);
12656		if (!event)
12657			break;
12658
12659		pmu_detach_event(pmu, event);
12660		put_event(event);
12661	}
12662
12663	/*
12664	 * wait for pending _free_event()s
12665	 */
12666	wait_var_event(pmu, pmu_empty(pmu));
12667}
12668
12669int perf_pmu_unregister(struct pmu *pmu)
12670{
12671	scoped_guard (mutex, &pmus_lock) {
12672		if (!idr_cmpxchg(&pmu_idr, pmu->type, pmu, NULL))
12673			return -EINVAL;
12674
12675		list_del_rcu(&pmu->entry);
12676	}
12677
12678	/*
12679	 * We dereference the pmu list under both SRCU and regular RCU, so
12680	 * synchronize against both of those.
12681	 *
12682	 * Notably, the entirety of event creation, from perf_init_event()
12683	 * (which will now fail, because of the above) until
12684	 * perf_install_in_context() should be under SRCU such that
12685	 * this synchronizes against event creation. This avoids trying to
12686	 * detach events that are not fully formed.
12687	 */
12688	synchronize_srcu(&pmus_srcu);
12689	synchronize_rcu();
12690
12691	if (pmu->event_unmapped && !pmu_empty(pmu)) {
12692		/*
12693		 * Can't force remove events when pmu::event_unmapped()
12694		 * is used in perf_mmap_close().
12695		 */
12696		guard(mutex)(&pmus_lock);
12697		idr_cmpxchg(&pmu_idr, pmu->type, NULL, pmu);
12698		list_add_rcu(&pmu->entry, &pmus);
12699		return -EBUSY;
12700	}
12701
12702	scoped_guard (mutex, &pmus_lock)
12703		idr_remove(&pmu_idr, pmu->type);
12704
12705	/*
12706	 * PMU is removed from the pmus list, so no new events will
12707	 * be created, now take care of the existing ones.
12708	 */
12709	pmu_detach_events(pmu);
12710
12711	/*
12712	 * PMU is unused, make it go away.
12713	 */
12714	perf_pmu_free(pmu);
12715	return 0;
12716}
12717EXPORT_SYMBOL_GPL(perf_pmu_unregister);
12718
12719static inline bool has_extended_regs(struct perf_event *event)
12720{
12721	return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) ||
12722	       (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK);
12723}
12724
12725static int perf_try_init_event(struct pmu *pmu, struct perf_event *event)
12726{
12727	struct perf_event_context *ctx = NULL;
12728	int ret;
12729
12730	if (!try_module_get(pmu->module))
12731		return -ENODEV;
12732
12733	/*
12734	 * A number of pmu->event_init() methods iterate the sibling_list to,
12735	 * for example, validate if the group fits on the PMU. Therefore,
12736	 * if this is a sibling event, acquire the ctx->mutex to protect
12737	 * the sibling_list.
12738	 */
12739	if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) {
12740		/*
12741		 * This ctx->mutex can nest when we're called through
12742		 * inheritance. See the perf_event_ctx_lock_nested() comment.
12743		 */
12744		ctx = perf_event_ctx_lock_nested(event->group_leader,
12745						 SINGLE_DEPTH_NESTING);
12746		BUG_ON(!ctx);
12747	}
12748
12749	event->pmu = pmu;
12750	ret = pmu->event_init(event);
12751
12752	if (ctx)
12753		perf_event_ctx_unlock(event->group_leader, ctx);
12754
12755	if (ret)
12756		goto err_pmu;
12757
12758	if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) &&
12759	    has_extended_regs(event)) {
12760		ret = -EOPNOTSUPP;
12761		goto err_destroy;
12762	}
12763
12764	if (pmu->capabilities & PERF_PMU_CAP_NO_EXCLUDE &&
12765	    event_has_any_exclude_flag(event)) {
12766		ret = -EINVAL;
12767		goto err_destroy;
12768	}
12769
12770	if (pmu->scope != PERF_PMU_SCOPE_NONE && event->cpu >= 0) {
12771		const struct cpumask *cpumask;
12772		struct cpumask *pmu_cpumask;
12773		int cpu;
12774
12775		cpumask = perf_scope_cpu_topology_cpumask(pmu->scope, event->cpu);
12776		pmu_cpumask = perf_scope_cpumask(pmu->scope);
12777
12778		ret = -ENODEV;
12779		if (!pmu_cpumask || !cpumask)
12780			goto err_destroy;
12781
12782		cpu = cpumask_any_and(pmu_cpumask, cpumask);
12783		if (cpu >= nr_cpu_ids)
12784			goto err_destroy;
12785
12786		event->event_caps |= PERF_EV_CAP_READ_SCOPE;
12787	}
12788
12789	return 0;
12790
12791err_destroy:
12792	if (event->destroy) {
12793		event->destroy(event);
12794		event->destroy = NULL;
12795	}
12796
12797err_pmu:
12798	event->pmu = NULL;
12799	module_put(pmu->module);
12800	return ret;
12801}
12802
12803static struct pmu *perf_init_event(struct perf_event *event)
12804{
12805	bool extended_type = false;
12806	struct pmu *pmu;
12807	int type, ret;
12808
12809	guard(srcu)(&pmus_srcu); /* pmu idr/list access */
12810
12811	/*
12812	 * Save original type before calling pmu->event_init() since certain
12813	 * pmus overwrites event->attr.type to forward event to another pmu.
12814	 */
12815	event->orig_type = event->attr.type;
12816
12817	/* Try parent's PMU first: */
12818	if (event->parent && event->parent->pmu) {
12819		pmu = event->parent->pmu;
12820		ret = perf_try_init_event(pmu, event);
12821		if (!ret)
12822			return pmu;
12823	}
12824
12825	/*
12826	 * PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE
12827	 * are often aliases for PERF_TYPE_RAW.
12828	 */
12829	type = event->attr.type;
12830	if (type == PERF_TYPE_HARDWARE || type == PERF_TYPE_HW_CACHE) {
12831		type = event->attr.config >> PERF_PMU_TYPE_SHIFT;
12832		if (!type) {
12833			type = PERF_TYPE_RAW;
12834		} else {
12835			extended_type = true;
12836			event->attr.config &= PERF_HW_EVENT_MASK;
12837		}
12838	}
12839
12840again:
12841	scoped_guard (rcu)
12842		pmu = idr_find(&pmu_idr, type);
12843	if (pmu) {
12844		if (event->attr.type != type && type != PERF_TYPE_RAW &&
12845		    !(pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE))
12846			return ERR_PTR(-ENOENT);
12847
12848		ret = perf_try_init_event(pmu, event);
12849		if (ret == -ENOENT && event->attr.type != type && !extended_type) {
12850			type = event->attr.type;
12851			goto again;
12852		}
12853
12854		if (ret)
12855			return ERR_PTR(ret);
12856
12857		return pmu;
12858	}
12859
12860	list_for_each_entry_rcu(pmu, &pmus, entry, lockdep_is_held(&pmus_srcu)) {
12861		ret = perf_try_init_event(pmu, event);
12862		if (!ret)
12863			return pmu;
12864
12865		if (ret != -ENOENT)
12866			return ERR_PTR(ret);
12867	}
12868
12869	return ERR_PTR(-ENOENT);
12870}
12871
12872static void attach_sb_event(struct perf_event *event)
12873{
12874	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);
12875
12876	raw_spin_lock(&pel->lock);
12877	list_add_rcu(&event->sb_list, &pel->list);
12878	raw_spin_unlock(&pel->lock);
12879}
12880
12881/*
12882 * We keep a list of all !task (and therefore per-cpu) events
12883 * that need to receive side-band records.
12884 *
12885 * This avoids having to scan all the various PMU per-cpu contexts
12886 * looking for them.
12887 */
12888static void account_pmu_sb_event(struct perf_event *event)
12889{
12890	if (is_sb_event(event))
12891		attach_sb_event(event);
12892}
12893
12894/* Freq events need the tick to stay alive (see perf_event_task_tick). */
12895static void account_freq_event_nohz(void)
12896{
12897#ifdef CONFIG_NO_HZ_FULL
12898	/* Lock so we don't race with concurrent unaccount */
12899	spin_lock(&nr_freq_lock);
12900	if (atomic_inc_return(&nr_freq_events) == 1)
12901		tick_nohz_dep_set(TICK_DEP_BIT_PERF_EVENTS);
12902	spin_unlock(&nr_freq_lock);
12903#endif
12904}
12905
12906static void account_freq_event(void)
12907{
12908	if (tick_nohz_full_enabled())
12909		account_freq_event_nohz();
12910	else
12911		atomic_inc(&nr_freq_events);
12912}
12913
12914
12915static void account_event(struct perf_event *event)
12916{
12917	bool inc = false;
12918
12919	if (event->parent)
12920		return;
12921
12922	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))
12923		inc = true;
12924	if (event->attr.mmap || event->attr.mmap_data)
12925		atomic_inc(&nr_mmap_events);
12926	if (event->attr.build_id)
12927		atomic_inc(&nr_build_id_events);
12928	if (event->attr.comm)
12929		atomic_inc(&nr_comm_events);
12930	if (event->attr.namespaces)
12931		atomic_inc(&nr_namespaces_events);
12932	if (event->attr.cgroup)
12933		atomic_inc(&nr_cgroup_events);
12934	if (event->attr.task)
12935		atomic_inc(&nr_task_events);
12936	if (event->attr.freq)
12937		account_freq_event();
12938	if (event->attr.context_switch) {
12939		atomic_inc(&nr_switch_events);
12940		inc = true;
12941	}
12942	if (has_branch_stack(event))
12943		inc = true;
12944	if (is_cgroup_event(event))
12945		inc = true;
12946	if (event->attr.ksymbol)
12947		atomic_inc(&nr_ksymbol_events);
12948	if (event->attr.bpf_event)
12949		atomic_inc(&nr_bpf_events);
12950	if (event->attr.text_poke)
12951		atomic_inc(&nr_text_poke_events);
12952
12953	if (inc) {
12954		/*
12955		 * We need the mutex here because static_branch_enable()
12956		 * must complete *before* the perf_sched_count increment
12957		 * becomes visible.
12958		 */
12959		if (atomic_inc_not_zero(&perf_sched_count))
12960			goto enabled;
12961
12962		mutex_lock(&perf_sched_mutex);
12963		if (!atomic_read(&perf_sched_count)) {
12964			static_branch_enable(&perf_sched_events);
12965			/*
12966			 * Guarantee that all CPUs observe they key change and
12967			 * call the perf scheduling hooks before proceeding to
12968			 * install events that need them.
12969			 */
12970			synchronize_rcu();
12971		}
12972		/*
12973		 * Now that we have waited for the sync_sched(), allow further
12974		 * increments to by-pass the mutex.
12975		 */
12976		atomic_inc(&perf_sched_count);
12977		mutex_unlock(&perf_sched_mutex);
12978	}
12979enabled:
12980
12981	account_pmu_sb_event(event);
12982}
12983
12984/*
12985 * Allocate and initialize an event structure
12986 */
12987static struct perf_event *
12988perf_event_alloc(struct perf_event_attr *attr, int cpu,
12989		 struct task_struct *task,
12990		 struct perf_event *group_leader,
12991		 struct perf_event *parent_event,
12992		 perf_overflow_handler_t overflow_handler,
12993		 void *context, int cgroup_fd)
12994{
12995	struct pmu *pmu;
12996	struct hw_perf_event *hwc;
12997	long err = -EINVAL;
12998	int node;
12999
13000	if ((unsigned)cpu >= nr_cpu_ids) {
13001		if (!task || cpu != -1)
13002			return ERR_PTR(-EINVAL);
13003	}
13004	if (attr->sigtrap && !task) {
13005		/* Requires a task: avoid signalling random tasks. */
13006		return ERR_PTR(-EINVAL);
13007	}
13008
13009	node = (cpu >= 0) ? cpu_to_node(cpu) : -1;
13010	struct perf_event *event __free(__free_event) =
13011		kmem_cache_alloc_node(perf_event_cache, GFP_KERNEL | __GFP_ZERO, node);
13012	if (!event)
13013		return ERR_PTR(-ENOMEM);
13014
13015	/*
13016	 * Single events are their own group leaders, with an
13017	 * empty sibling list:
13018	 */
13019	if (!group_leader)
13020		group_leader = event;
13021
13022	mutex_init(&event->child_mutex);
13023	INIT_LIST_HEAD(&event->child_list);
13024
13025	INIT_LIST_HEAD(&event->event_entry);
13026	INIT_LIST_HEAD(&event->sibling_list);
13027	INIT_LIST_HEAD(&event->active_list);
13028	init_event_group(event);
13029	INIT_LIST_HEAD(&event->rb_entry);
13030	INIT_LIST_HEAD(&event->active_entry);
13031	INIT_LIST_HEAD(&event->addr_filters.list);
13032	INIT_HLIST_NODE(&event->hlist_entry);
13033	INIT_LIST_HEAD(&event->pmu_list);
13034
13035
13036	init_waitqueue_head(&event->waitq);
13037	init_irq_work(&event->pending_irq, perf_pending_irq);
13038	event->pending_disable_irq = IRQ_WORK_INIT_HARD(perf_pending_disable);
13039	init_task_work(&event->pending_task, perf_pending_task);
13040
13041	mutex_init(&event->mmap_mutex);
13042	raw_spin_lock_init(&event->addr_filters.lock);
13043
13044	atomic_long_set(&event->refcount, 1);
13045	event->cpu		= cpu;
13046	event->attr		= *attr;
13047	event->group_leader	= group_leader;
13048	event->pmu		= NULL;
13049	event->oncpu		= -1;
13050
13051	event->parent		= parent_event;
13052
13053	event->ns		= get_pid_ns(task_active_pid_ns(current));
13054	event->id		= atomic64_inc_return(&perf_event_id);
13055
13056	event->state		= PERF_EVENT_STATE_INACTIVE;
13057
13058	if (parent_event)
13059		event->event_caps = parent_event->event_caps;
13060
13061	if (task) {
13062		event->attach_state = PERF_ATTACH_TASK;
13063		/*
13064		 * XXX pmu::event_init needs to know what task to account to
13065		 * and we cannot use the ctx information because we need the
13066		 * pmu before we get a ctx.
13067		 */
13068		event->hw.target = get_task_struct(task);
13069	}
13070
13071	event->clock = &local_clock;
13072	if (parent_event)
13073		event->clock = parent_event->clock;
13074
13075	if (!overflow_handler && parent_event) {
13076		overflow_handler = parent_event->overflow_handler;
13077		context = parent_event->overflow_handler_context;
13078#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_EVENT_TRACING)
13079		if (parent_event->prog) {
13080			struct bpf_prog *prog = parent_event->prog;
13081
13082			bpf_prog_inc(prog);
13083			event->prog = prog;
13084		}
13085#endif
13086	}
13087
13088	if (overflow_handler) {
13089		event->overflow_handler	= overflow_handler;
13090		event->overflow_handler_context = context;
13091	} else if (is_write_backward(event)){
13092		event->overflow_handler = perf_event_output_backward;
13093		event->overflow_handler_context = NULL;
13094	} else {
13095		event->overflow_handler = perf_event_output_forward;
13096		event->overflow_handler_context = NULL;
13097	}
13098
13099	perf_event__state_init(event);
13100
13101	pmu = NULL;
13102
13103	hwc = &event->hw;
13104	hwc->sample_period = attr->sample_period;
13105	if (is_event_in_freq_mode(event))
13106		hwc->sample_period = 1;
13107	hwc->last_period = hwc->sample_period;
13108
13109	local64_set(&hwc->period_left, hwc->sample_period);
13110
13111	/*
13112	 * We do not support PERF_SAMPLE_READ on inherited events unless
13113	 * PERF_SAMPLE_TID is also selected, which allows inherited events to
13114	 * collect per-thread samples.
13115	 * See perf_output_read().
13116	 */
13117	if (has_inherit_and_sample_read(attr) && !(attr->sample_type & PERF_SAMPLE_TID))
13118		return ERR_PTR(-EINVAL);
13119
13120	if (!has_branch_stack(event))
13121		event->attr.branch_sample_type = 0;
13122
13123	pmu = perf_init_event(event);
13124	if (IS_ERR(pmu))
13125		return (void*)pmu;
13126
13127	/*
13128	 * The PERF_ATTACH_TASK_DATA is set in the event_init()->hw_config().
13129	 * The attach should be right after the perf_init_event().
13130	 * Otherwise, the __free_event() would mistakenly detach the non-exist
13131	 * perf_ctx_data because of the other errors between them.
13132	 */
13133	if (event->attach_state & PERF_ATTACH_TASK_DATA) {
13134		err = attach_perf_ctx_data(event);
13135		if (err)
13136			return ERR_PTR(err);
13137	}
13138
13139	/*
13140	 * Disallow uncore-task events. Similarly, disallow uncore-cgroup
13141	 * events (they don't make sense as the cgroup will be different
13142	 * on other CPUs in the uncore mask).
13143	 */
13144	if (pmu->task_ctx_nr == perf_invalid_context && (task || cgroup_fd != -1))
13145		return ERR_PTR(-EINVAL);
13146
13147	if (event->attr.aux_output &&
13148	    (!(pmu->capabilities & PERF_PMU_CAP_AUX_OUTPUT) ||
13149	     event->attr.aux_pause || event->attr.aux_resume))
13150		return ERR_PTR(-EOPNOTSUPP);
13151
13152	if (event->attr.aux_pause && event->attr.aux_resume)
13153		return ERR_PTR(-EINVAL);
13154
13155	if (event->attr.aux_start_paused) {
13156		if (!(pmu->capabilities & PERF_PMU_CAP_AUX_PAUSE))
13157			return ERR_PTR(-EOPNOTSUPP);
13158		event->hw.aux_paused = 1;
13159	}
13160
13161	if (cgroup_fd != -1) {
13162		err = perf_cgroup_connect(cgroup_fd, event, attr, group_leader);
13163		if (err)
13164			return ERR_PTR(err);
13165	}
13166
13167	err = exclusive_event_init(event);
13168	if (err)
13169		return ERR_PTR(err);
13170
13171	if (has_addr_filter(event)) {
13172		event->addr_filter_ranges = kcalloc(pmu->nr_addr_filters,
13173						    sizeof(struct perf_addr_filter_range),
13174						    GFP_KERNEL);
13175		if (!event->addr_filter_ranges)
13176			return ERR_PTR(-ENOMEM);
13177
13178		/*
13179		 * Clone the parent's vma offsets: they are valid until exec()
13180		 * even if the mm is not shared with the parent.
13181		 */
13182		if (event->parent) {
13183			struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
13184
13185			raw_spin_lock_irq(&ifh->lock);
13186			memcpy(event->addr_filter_ranges,
13187			       event->parent->addr_filter_ranges,
13188			       pmu->nr_addr_filters * sizeof(struct perf_addr_filter_range));
13189			raw_spin_unlock_irq(&ifh->lock);
13190		}
13191
13192		/* force hw sync on the address filters */
13193		event->addr_filters_gen = 1;
13194	}
13195
13196	if (!event->parent) {
13197		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
13198			err = get_callchain_buffers(attr->sample_max_stack);
13199			if (err)
13200				return ERR_PTR(err);
13201			event->attach_state |= PERF_ATTACH_CALLCHAIN;
13202		}
13203	}
13204
13205	err = security_perf_event_alloc(event);
13206	if (err)
13207		return ERR_PTR(err);
13208
13209	/* symmetric to unaccount_event() in _free_event() */
13210	account_event(event);
13211
13212	/*
13213	 * Event creation should be under SRCU, see perf_pmu_unregister().
13214	 */
13215	lockdep_assert_held(&pmus_srcu);
13216	scoped_guard (spinlock, &pmu->events_lock)
13217		list_add(&event->pmu_list, &pmu->events);
13218
13219	return_ptr(event);
13220}
13221
13222static int perf_copy_attr(struct perf_event_attr __user *uattr,
13223			  struct perf_event_attr *attr)
13224{
13225	u32 size;
13226	int ret;
13227
13228	/* Zero the full structure, so that a short copy will be nice. */
13229	memset(attr, 0, sizeof(*attr));
13230
13231	ret = get_user(size, &uattr->size);
13232	if (ret)
13233		return ret;
13234
13235	/* ABI compatibility quirk: */
13236	if (!size)
13237		size = PERF_ATTR_SIZE_VER0;
13238	if (size < PERF_ATTR_SIZE_VER0 || size > PAGE_SIZE)
13239		goto err_size;
13240
13241	ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size);
13242	if (ret) {
13243		if (ret == -E2BIG)
13244			goto err_size;
13245		return ret;
13246	}
13247
13248	attr->size = size;
13249
13250	if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3)
13251		return -EINVAL;
13252
13253	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
13254		return -EINVAL;
13255
13256	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
13257		return -EINVAL;
13258
13259	if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
13260		u64 mask = attr->branch_sample_type;
13261
13262		/* only using defined bits */
13263		if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
13264			return -EINVAL;
13265
13266		/* at least one branch bit must be set */
13267		if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
13268			return -EINVAL;
13269
13270		/* propagate priv level, when not set for branch */
13271		if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
13272
13273			/* exclude_kernel checked on syscall entry */
13274			if (!attr->exclude_kernel)
13275				mask |= PERF_SAMPLE_BRANCH_KERNEL;
13276
13277			if (!attr->exclude_user)
13278				mask |= PERF_SAMPLE_BRANCH_USER;
13279
13280			if (!attr->exclude_hv)
13281				mask |= PERF_SAMPLE_BRANCH_HV;
13282			/*
13283			 * adjust user setting (for HW filter setup)
13284			 */
13285			attr->branch_sample_type = mask;
13286		}
13287		/* privileged levels capture (kernel, hv): check permissions */
13288		if (mask & PERF_SAMPLE_BRANCH_PERM_PLM) {
13289			ret = perf_allow_kernel();
13290			if (ret)
13291				return ret;
13292		}
13293	}
13294
13295	if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
13296		ret = perf_reg_validate(attr->sample_regs_user);
13297		if (ret)
13298			return ret;
13299	}
13300
13301	if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
13302		if (!arch_perf_have_user_stack_dump())
13303			return -ENOSYS;
13304
13305		/*
13306		 * We have __u32 type for the size, but so far
13307		 * we can only use __u16 as maximum due to the
13308		 * __u16 sample size limit.
13309		 */
13310		if (attr->sample_stack_user >= USHRT_MAX)
13311			return -EINVAL;
13312		else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))
13313			return -EINVAL;
13314	}
13315
13316	if (!attr->sample_max_stack)
13317		attr->sample_max_stack = sysctl_perf_event_max_stack;
13318
13319	if (attr->sample_type & PERF_SAMPLE_REGS_INTR)
13320		ret = perf_reg_validate(attr->sample_regs_intr);
13321
13322#ifndef CONFIG_CGROUP_PERF
13323	if (attr->sample_type & PERF_SAMPLE_CGROUP)
13324		return -EINVAL;
13325#endif
13326	if ((attr->sample_type & PERF_SAMPLE_WEIGHT) &&
13327	    (attr->sample_type & PERF_SAMPLE_WEIGHT_STRUCT))
13328		return -EINVAL;
13329
13330	if (!attr->inherit && attr->inherit_thread)
13331		return -EINVAL;
13332
13333	if (attr->remove_on_exec && attr->enable_on_exec)
13334		return -EINVAL;
13335
13336	if (attr->sigtrap && !attr->remove_on_exec)
13337		return -EINVAL;
13338
13339out:
13340	return ret;
13341
13342err_size:
13343	put_user(sizeof(*attr), &uattr->size);
13344	ret = -E2BIG;
13345	goto out;
13346}
13347
13348static void mutex_lock_double(struct mutex *a, struct mutex *b)
13349{
13350	if (b < a)
13351		swap(a, b);
13352
13353	mutex_lock(a);
13354	mutex_lock_nested(b, SINGLE_DEPTH_NESTING);
13355}
13356
13357static int
13358perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
13359{
13360	struct perf_buffer *rb = NULL;
13361	int ret = -EINVAL;
13362
13363	if (!output_event) {
13364		mutex_lock(&event->mmap_mutex);
13365		goto set;
13366	}
13367
13368	/* don't allow circular references */
13369	if (event == output_event)
13370		goto out;
13371
13372	/*
13373	 * Don't allow cross-cpu buffers
13374	 */
13375	if (output_event->cpu != event->cpu)
13376		goto out;
13377
13378	/*
13379	 * If its not a per-cpu rb, it must be the same task.
13380	 */
13381	if (output_event->cpu == -1 && output_event->hw.target != event->hw.target)
13382		goto out;
13383
13384	/*
13385	 * Mixing clocks in the same buffer is trouble you don't need.
13386	 */
13387	if (output_event->clock != event->clock)
13388		goto out;
13389
13390	/*
13391	 * Either writing ring buffer from beginning or from end.
13392	 * Mixing is not allowed.
13393	 */
13394	if (is_write_backward(output_event) != is_write_backward(event))
13395		goto out;
13396
13397	/*
13398	 * If both events generate aux data, they must be on the same PMU
13399	 */
13400	if (has_aux(event) && has_aux(output_event) &&
13401	    event->pmu != output_event->pmu)
13402		goto out;
13403
13404	/*
13405	 * Hold both mmap_mutex to serialize against perf_mmap_close().  Since
13406	 * output_event is already on rb->event_list, and the list iteration
13407	 * restarts after every removal, it is guaranteed this new event is
13408	 * observed *OR* if output_event is already removed, it's guaranteed we
13409	 * observe !rb->mmap_count.
13410	 */
13411	mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex);
13412set:
13413	/* Can't redirect output if we've got an active mmap() */
13414	if (refcount_read(&event->mmap_count))
13415		goto unlock;
13416
13417	if (output_event) {
13418		if (output_event->state <= PERF_EVENT_STATE_REVOKED)
13419			goto unlock;
13420
13421		/* get the rb we want to redirect to */
13422		rb = ring_buffer_get(output_event);
13423		if (!rb)
13424			goto unlock;
13425
13426		/* did we race against perf_mmap_close() */
13427		if (!refcount_read(&rb->mmap_count)) {
13428			ring_buffer_put(rb);
13429			goto unlock;
13430		}
13431	}
13432
13433	ring_buffer_attach(event, rb);
13434
13435	ret = 0;
13436unlock:
13437	mutex_unlock(&event->mmap_mutex);
13438	if (output_event)
13439		mutex_unlock(&output_event->mmap_mutex);
13440
13441out:
13442	return ret;
13443}
13444
13445static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id)
13446{
13447	bool nmi_safe = false;
13448
13449	switch (clk_id) {
13450	case CLOCK_MONOTONIC:
13451		event->clock = &ktime_get_mono_fast_ns;
13452		nmi_safe = true;
13453		break;
13454
13455	case CLOCK_MONOTONIC_RAW:
13456		event->clock = &ktime_get_raw_fast_ns;
13457		nmi_safe = true;
13458		break;
13459
13460	case CLOCK_REALTIME:
13461		event->clock = &ktime_get_real_ns;
13462		break;
13463
13464	case CLOCK_BOOTTIME:
13465		event->clock = &ktime_get_boottime_ns;
13466		break;
13467
13468	case CLOCK_TAI:
13469		event->clock = &ktime_get_clocktai_ns;
13470		break;
13471
13472	default:
13473		return -EINVAL;
13474	}
13475
13476	if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI))
13477		return -EINVAL;
13478
13479	return 0;
13480}
13481
13482static bool
13483perf_check_permission(struct perf_event_attr *attr, struct task_struct *task)
13484{
13485	unsigned int ptrace_mode = PTRACE_MODE_READ_REALCREDS;
13486	bool is_capable = perfmon_capable();
13487
13488	if (attr->sigtrap) {
13489		/*
13490		 * perf_event_attr::sigtrap sends signals to the other task.
13491		 * Require the current task to also have CAP_KILL.
13492		 */
13493		rcu_read_lock();
13494		is_capable &= ns_capable(__task_cred(task)->user_ns, CAP_KILL);
13495		rcu_read_unlock();
13496
13497		/*
13498		 * If the required capabilities aren't available, checks for
13499		 * ptrace permissions: upgrade to ATTACH, since sending signals
13500		 * can effectively change the target task.
13501		 */
13502		ptrace_mode = PTRACE_MODE_ATTACH_REALCREDS;
13503	}
13504
13505	/*
13506	 * Preserve ptrace permission check for backwards compatibility. The
13507	 * ptrace check also includes checks that the current task and other
13508	 * task have matching uids, and is therefore not done here explicitly.
13509	 */
13510	return is_capable || ptrace_may_access(task, ptrace_mode);
13511}
13512
13513/**
13514 * sys_perf_event_open - open a performance event, associate it to a task/cpu
13515 *
13516 * @attr_uptr:	event_id type attributes for monitoring/sampling
13517 * @pid:		target pid
13518 * @cpu:		target cpu
13519 * @group_fd:		group leader event fd
13520 * @flags:		perf event open flags
13521 */
13522SYSCALL_DEFINE5(perf_event_open,
13523		struct perf_event_attr __user *, attr_uptr,
13524		pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
13525{
13526	struct perf_event *group_leader = NULL, *output_event = NULL;
13527	struct perf_event_pmu_context *pmu_ctx;
13528	struct perf_event *event, *sibling;
13529	struct perf_event_attr attr;
13530	struct perf_event_context *ctx;
13531	struct file *event_file = NULL;
13532	struct task_struct *task = NULL;
13533	struct pmu *pmu;
13534	int event_fd;
13535	int move_group = 0;
13536	int err;
13537	int f_flags = O_RDWR;
13538	int cgroup_fd = -1;
13539
13540	/* for future expandability... */
13541	if (flags & ~PERF_FLAG_ALL)
13542		return -EINVAL;
13543
13544	err = perf_copy_attr(attr_uptr, &attr);
13545	if (err)
13546		return err;
13547
13548	/* Do we allow access to perf_event_open(2) ? */
13549	err = security_perf_event_open(PERF_SECURITY_OPEN);
13550	if (err)
13551		return err;
13552
13553	if (!attr.exclude_kernel) {
13554		err = perf_allow_kernel();
13555		if (err)
13556			return err;
13557	}
13558
13559	if (attr.namespaces) {
13560		if (!perfmon_capable())
13561			return -EACCES;
13562	}
13563
13564	if (attr.freq) {
13565		if (attr.sample_freq > sysctl_perf_event_sample_rate)
13566			return -EINVAL;
13567	} else {
13568		if (attr.sample_period & (1ULL << 63))
13569			return -EINVAL;
13570	}
13571
13572	/* Only privileged users can get physical addresses */
13573	if ((attr.sample_type & PERF_SAMPLE_PHYS_ADDR)) {
13574		err = perf_allow_kernel();
13575		if (err)
13576			return err;
13577	}
13578
13579	/* REGS_INTR can leak data, lockdown must prevent this */
13580	if (attr.sample_type & PERF_SAMPLE_REGS_INTR) {
13581		err = security_locked_down(LOCKDOWN_PERF);
13582		if (err)
13583			return err;
13584	}
13585
13586	/*
13587	 * In cgroup mode, the pid argument is used to pass the fd
13588	 * opened to the cgroup directory in cgroupfs. The cpu argument
13589	 * designates the cpu on which to monitor threads from that
13590	 * cgroup.
13591	 */
13592	if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
13593		return -EINVAL;
13594
13595	if (flags & PERF_FLAG_FD_CLOEXEC)
13596		f_flags |= O_CLOEXEC;
13597
13598	event_fd = get_unused_fd_flags(f_flags);
13599	if (event_fd < 0)
13600		return event_fd;
13601
13602	/*
13603	 * Event creation should be under SRCU, see perf_pmu_unregister().
13604	 */
13605	guard(srcu)(&pmus_srcu);
13606
13607	CLASS(fd, group)(group_fd);     // group_fd == -1 => empty
13608	if (group_fd != -1) {
13609		if (!is_perf_file(group)) {
13610			err = -EBADF;
13611			goto err_fd;
13612		}
13613		group_leader = fd_file(group)->private_data;
13614		if (group_leader->state <= PERF_EVENT_STATE_REVOKED) {
13615			err = -ENODEV;
13616			goto err_fd;
13617		}
13618		if (flags & PERF_FLAG_FD_OUTPUT)
13619			output_event = group_leader;
13620		if (flags & PERF_FLAG_FD_NO_GROUP)
13621			group_leader = NULL;
13622	}
13623
13624	if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
13625		task = find_lively_task_by_vpid(pid);
13626		if (IS_ERR(task)) {
13627			err = PTR_ERR(task);
13628			goto err_fd;
13629		}
13630	}
13631
13632	if (task && group_leader &&
13633	    group_leader->attr.inherit != attr.inherit) {
13634		err = -EINVAL;
13635		goto err_task;
13636	}
13637
13638	if (flags & PERF_FLAG_PID_CGROUP)
13639		cgroup_fd = pid;
13640
13641	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
13642				 NULL, NULL, cgroup_fd);
13643	if (IS_ERR(event)) {
13644		err = PTR_ERR(event);
13645		goto err_task;
13646	}
13647
13648	if (is_sampling_event(event)) {
13649		if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {
13650			err = -EOPNOTSUPP;
13651			goto err_alloc;
13652		}
13653	}
13654
13655	/*
13656	 * Special case software events and allow them to be part of
13657	 * any hardware group.
13658	 */
13659	pmu = event->pmu;
13660
13661	if (attr.use_clockid) {
13662		err = perf_event_set_clock(event, attr.clockid);
13663		if (err)
13664			goto err_alloc;
13665	}
13666
13667	if (pmu->task_ctx_nr == perf_sw_context)
13668		event->event_caps |= PERF_EV_CAP_SOFTWARE;
13669
13670	if (task) {
13671		err = down_read_interruptible(&task->signal->exec_update_lock);
13672		if (err)
13673			goto err_alloc;
13674
13675		/*
13676		 * We must hold exec_update_lock across this and any potential
13677		 * perf_install_in_context() call for this new event to
13678		 * serialize against exec() altering our credentials (and the
13679		 * perf_event_exit_task() that could imply).
13680		 */
13681		err = -EACCES;
13682		if (!perf_check_permission(&attr, task))
13683			goto err_cred;
13684	}
13685
13686	/*
13687	 * Get the target context (task or percpu):
13688	 */
13689	ctx = find_get_context(task, event);
13690	if (IS_ERR(ctx)) {
13691		err = PTR_ERR(ctx);
13692		goto err_cred;
13693	}
13694
13695	mutex_lock(&ctx->mutex);
13696
13697	if (ctx->task == TASK_TOMBSTONE) {
13698		err = -ESRCH;
13699		goto err_locked;
13700	}
13701
13702	if (!task) {
13703		/*
13704		 * Check if the @cpu we're creating an event for is online.
13705		 *
13706		 * We use the perf_cpu_context::ctx::mutex to serialize against
13707		 * the hotplug notifiers. See perf_event_{init,exit}_cpu().
13708		 */
13709		struct perf_cpu_context *cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);
13710
13711		if (!cpuctx->online) {
13712			err = -ENODEV;
13713			goto err_locked;
13714		}
13715	}
13716
13717	if (group_leader) {
13718		err = -EINVAL;
13719
13720		/*
13721		 * Do not allow a recursive hierarchy (this new sibling
13722		 * becoming part of another group-sibling):
13723		 */
13724		if (group_leader->group_leader != group_leader)
13725			goto err_locked;
13726
13727		/* All events in a group should have the same clock */
13728		if (group_leader->clock != event->clock)
13729			goto err_locked;
13730
13731		/*
13732		 * Make sure we're both events for the same CPU;
13733		 * grouping events for different CPUs is broken; since
13734		 * you can never concurrently schedule them anyhow.
13735		 */
13736		if (group_leader->cpu != event->cpu)
13737			goto err_locked;
13738
13739		/*
13740		 * Make sure we're both on the same context; either task or cpu.
13741		 */
13742		if (group_leader->ctx != ctx)
13743			goto err_locked;
13744
13745		/*
13746		 * Only a group leader can be exclusive or pinned
13747		 */
13748		if (attr.exclusive || attr.pinned)
13749			goto err_locked;
13750
13751		if (is_software_event(event) &&
13752		    !in_software_context(group_leader)) {
13753			/*
13754			 * If the event is a sw event, but the group_leader
13755			 * is on hw context.
13756			 *
13757			 * Allow the addition of software events to hw
13758			 * groups, this is safe because software events
13759			 * never fail to schedule.
13760			 *
13761			 * Note the comment that goes with struct
13762			 * perf_event_pmu_context.
13763			 */
13764			pmu = group_leader->pmu_ctx->pmu;
13765		} else if (!is_software_event(event)) {
13766			if (is_software_event(group_leader) &&
13767			    (group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) {
13768				/*
13769				 * In case the group is a pure software group, and we
13770				 * try to add a hardware event, move the whole group to
13771				 * the hardware context.
13772				 */
13773				move_group = 1;
13774			}
13775
13776			/* Don't allow group of multiple hw events from different pmus */
13777			if (!in_software_context(group_leader) &&
13778			    group_leader->pmu_ctx->pmu != pmu)
13779				goto err_locked;
13780		}
13781	}
13782
13783	/*
13784	 * Now that we're certain of the pmu; find the pmu_ctx.
13785	 */
13786	pmu_ctx = find_get_pmu_context(pmu, ctx, event);
13787	if (IS_ERR(pmu_ctx)) {
13788		err = PTR_ERR(pmu_ctx);
13789		goto err_locked;
13790	}
13791	event->pmu_ctx = pmu_ctx;
13792
13793	if (output_event) {
13794		err = perf_event_set_output(event, output_event);
13795		if (err)
13796			goto err_context;
13797	}
13798
13799	if (!perf_event_validate_size(event)) {
13800		err = -E2BIG;
13801		goto err_context;
13802	}
13803
13804	if (perf_need_aux_event(event) && !perf_get_aux_event(event, group_leader)) {
13805		err = -EINVAL;
13806		goto err_context;
13807	}
13808
13809	/*
13810	 * Must be under the same ctx::mutex as perf_install_in_context(),
13811	 * because we need to serialize with concurrent event creation.
13812	 */
13813	if (!exclusive_event_installable(event, ctx)) {
13814		err = -EBUSY;
13815		goto err_context;
13816	}
13817
13818	WARN_ON_ONCE(ctx->parent_ctx);
13819
13820	event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, f_flags);
13821	if (IS_ERR(event_file)) {
13822		err = PTR_ERR(event_file);
13823		event_file = NULL;
13824		goto err_context;
13825	}
13826
13827	/*
13828	 * This is the point on no return; we cannot fail hereafter. This is
13829	 * where we start modifying current state.
13830	 */
13831
13832	if (move_group) {
13833		perf_remove_from_context(group_leader, 0);
13834		put_pmu_ctx(group_leader->pmu_ctx);
13835
13836		for_each_sibling_event(sibling, group_leader) {
13837			perf_remove_from_context(sibling, 0);
13838			put_pmu_ctx(sibling->pmu_ctx);
13839		}
13840
13841		/*
13842		 * Install the group siblings before the group leader.
13843		 *
13844		 * Because a group leader will try and install the entire group
13845		 * (through the sibling list, which is still in-tact), we can
13846		 * end up with siblings installed in the wrong context.
13847		 *
13848		 * By installing siblings first we NO-OP because they're not
13849		 * reachable through the group lists.
13850		 */
13851		for_each_sibling_event(sibling, group_leader) {
13852			sibling->pmu_ctx = pmu_ctx;
13853			get_pmu_ctx(pmu_ctx);
13854			perf_event__state_init(sibling);
13855			perf_install_in_context(ctx, sibling, sibling->cpu);
13856		}
13857
13858		/*
13859		 * Removing from the context ends up with disabled
13860		 * event. What we want here is event in the initial
13861		 * startup state, ready to be add into new context.
13862		 */
13863		group_leader->pmu_ctx = pmu_ctx;
13864		get_pmu_ctx(pmu_ctx);
13865		perf_event__state_init(group_leader);
13866		perf_install_in_context(ctx, group_leader, group_leader->cpu);
13867	}
13868
13869	/*
13870	 * Precalculate sample_data sizes; do while holding ctx::mutex such
13871	 * that we're serialized against further additions and before
13872	 * perf_install_in_context() which is the point the event is active and
13873	 * can use these values.
13874	 */
13875	perf_event__header_size(event);
13876	perf_event__id_header_size(event);
13877
13878	event->owner = current;
13879
13880	perf_install_in_context(ctx, event, event->cpu);
13881	perf_unpin_context(ctx);
13882
13883	mutex_unlock(&ctx->mutex);
13884
13885	if (task) {
13886		up_read(&task->signal->exec_update_lock);
13887		put_task_struct(task);
13888	}
13889
13890	mutex_lock(&current->perf_event_mutex);
13891	list_add_tail(&event->owner_entry, &current->perf_event_list);
13892	mutex_unlock(&current->perf_event_mutex);
13893
13894	/*
13895	 * File reference in group guarantees that group_leader has been
13896	 * kept alive until we place the new event on the sibling_list.
13897	 * This ensures destruction of the group leader will find
13898	 * the pointer to itself in perf_group_detach().
13899	 */
13900	fd_install(event_fd, event_file);
13901	return event_fd;
13902
13903err_context:
13904	put_pmu_ctx(event->pmu_ctx);
13905	event->pmu_ctx = NULL; /* _free_event() */
13906err_locked:
13907	mutex_unlock(&ctx->mutex);
13908	perf_unpin_context(ctx);
13909	put_ctx(ctx);
13910err_cred:
13911	if (task)
13912		up_read(&task->signal->exec_update_lock);
13913err_alloc:
13914	put_event(event);
13915err_task:
13916	if (task)
13917		put_task_struct(task);
13918err_fd:
13919	put_unused_fd(event_fd);
13920	return err;
13921}
13922
13923/**
13924 * perf_event_create_kernel_counter
13925 *
13926 * @attr: attributes of the counter to create
13927 * @cpu: cpu in which the counter is bound
13928 * @task: task to profile (NULL for percpu)
13929 * @overflow_handler: callback to trigger when we hit the event
13930 * @context: context data could be used in overflow_handler callback
13931 */
13932struct perf_event *
13933perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
13934				 struct task_struct *task,
13935				 perf_overflow_handler_t overflow_handler,
13936				 void *context)
13937{
13938	struct perf_event_pmu_context *pmu_ctx;
13939	struct perf_event_context *ctx;
13940	struct perf_event *event;
13941	struct pmu *pmu;
13942	int err;
13943
13944	/*
13945	 * Grouping is not supported for kernel events, neither is 'AUX',
13946	 * make sure the caller's intentions are adjusted.
13947	 */
13948	if (attr->aux_output || attr->aux_action)
13949		return ERR_PTR(-EINVAL);
13950
13951	/*
13952	 * Event creation should be under SRCU, see perf_pmu_unregister().
13953	 */
13954	guard(srcu)(&pmus_srcu);
13955
13956	event = perf_event_alloc(attr, cpu, task, NULL, NULL,
13957				 overflow_handler, context, -1);
13958	if (IS_ERR(event)) {
13959		err = PTR_ERR(event);
13960		goto err;
13961	}
13962
13963	/* Mark owner so we could distinguish it from user events. */
13964	event->owner = TASK_TOMBSTONE;
13965	pmu = event->pmu;
13966
13967	if (pmu->task_ctx_nr == perf_sw_context)
13968		event->event_caps |= PERF_EV_CAP_SOFTWARE;
13969
13970	/*
13971	 * Get the target context (task or percpu):
13972	 */
13973	ctx = find_get_context(task, event);
13974	if (IS_ERR(ctx)) {
13975		err = PTR_ERR(ctx);
13976		goto err_alloc;
13977	}
13978
13979	WARN_ON_ONCE(ctx->parent_ctx);
13980	mutex_lock(&ctx->mutex);
13981	if (ctx->task == TASK_TOMBSTONE) {
13982		err = -ESRCH;
13983		goto err_unlock;
13984	}
13985
13986	pmu_ctx = find_get_pmu_context(pmu, ctx, event);
13987	if (IS_ERR(pmu_ctx)) {
13988		err = PTR_ERR(pmu_ctx);
13989		goto err_unlock;
13990	}
13991	event->pmu_ctx = pmu_ctx;
13992
13993	if (!task) {
13994		/*
13995		 * Check if the @cpu we're creating an event for is online.
13996		 *
13997		 * We use the perf_cpu_context::ctx::mutex to serialize against
13998		 * the hotplug notifiers. See perf_event_{init,exit}_cpu().
13999		 */
14000		struct perf_cpu_context *cpuctx =
14001			container_of(ctx, struct perf_cpu_context, ctx);
14002		if (!cpuctx->online) {
14003			err = -ENODEV;
14004			goto err_pmu_ctx;
14005		}
14006	}
14007
14008	if (!exclusive_event_installable(event, ctx)) {
14009		err = -EBUSY;
14010		goto err_pmu_ctx;
14011	}
14012
14013	perf_install_in_context(ctx, event, event->cpu);
14014	perf_unpin_context(ctx);
14015	mutex_unlock(&ctx->mutex);
14016
14017	return event;
14018
14019err_pmu_ctx:
14020	put_pmu_ctx(pmu_ctx);
14021	event->pmu_ctx = NULL; /* _free_event() */
14022err_unlock:
14023	mutex_unlock(&ctx->mutex);
14024	perf_unpin_context(ctx);
14025	put_ctx(ctx);
14026err_alloc:
14027	put_event(event);
14028err:
14029	return ERR_PTR(err);
14030}
14031EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
14032
14033static void __perf_pmu_remove(struct perf_event_context *ctx,
14034			      int cpu, struct pmu *pmu,
14035			      struct perf_event_groups *groups,
14036			      struct list_head *events)
14037{
14038	struct perf_event *event, *sibling;
14039
14040	perf_event_groups_for_cpu_pmu(event, groups, cpu, pmu) {
14041		perf_remove_from_context(event, 0);
14042		put_pmu_ctx(event->pmu_ctx);
14043		list_add(&event->migrate_entry, events);
14044
14045		for_each_sibling_event(sibling, event) {
14046			perf_remove_from_context(sibling, 0);
14047			put_pmu_ctx(sibling->pmu_ctx);
14048			list_add(&sibling->migrate_entry, events);
14049		}
14050	}
14051}
14052
14053static void __perf_pmu_install_event(struct pmu *pmu,
14054				     struct perf_event_context *ctx,
14055				     int cpu, struct perf_event *event)
14056{
14057	struct perf_event_pmu_context *epc;
14058	struct perf_event_context *old_ctx = event->ctx;
14059
14060	get_ctx(ctx); /* normally find_get_context() */
14061
14062	event->cpu = cpu;
14063	epc = find_get_pmu_context(pmu, ctx, event);
14064	event->pmu_ctx = epc;
14065
14066	if (event->state >= PERF_EVENT_STATE_OFF)
14067		event->state = PERF_EVENT_STATE_INACTIVE;
14068	perf_install_in_context(ctx, event, cpu);
14069
14070	/*
14071	 * Now that event->ctx is updated and visible, put the old ctx.
14072	 */
14073	put_ctx(old_ctx);
14074}
14075
14076static void __perf_pmu_install(struct perf_event_context *ctx,
14077			       int cpu, struct pmu *pmu, struct list_head *events)
14078{
14079	struct perf_event *event, *tmp;
14080
14081	/*
14082	 * Re-instate events in 2 passes.
14083	 *
14084	 * Skip over group leaders and only install siblings on this first
14085	 * pass, siblings will not get enabled without a leader, however a
14086	 * leader will enable its siblings, even if those are still on the old
14087	 * context.
14088	 */
14089	list_for_each_entry_safe(event, tmp, events, migrate_entry) {
14090		if (event->group_leader == event)
14091			continue;
14092
14093		list_del(&event->migrate_entry);
14094		__perf_pmu_install_event(pmu, ctx, cpu, event);
14095	}
14096
14097	/*
14098	 * Once all the siblings are setup properly, install the group leaders
14099	 * to make it go.
14100	 */
14101	list_for_each_entry_safe(event, tmp, events, migrate_entry) {
14102		list_del(&event->migrate_entry);
14103		__perf_pmu_install_event(pmu, ctx, cpu, event);
14104	}
14105}
14106
14107void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
14108{
14109	struct perf_event_context *src_ctx, *dst_ctx;
14110	LIST_HEAD(events);
14111
14112	/*
14113	 * Since per-cpu context is persistent, no need to grab an extra
14114	 * reference.
14115	 */
14116	src_ctx = &per_cpu_ptr(&perf_cpu_context, src_cpu)->ctx;
14117	dst_ctx = &per_cpu_ptr(&perf_cpu_context, dst_cpu)->ctx;
14118
14119	/*
14120	 * See perf_event_ctx_lock() for comments on the details
14121	 * of swizzling perf_event::ctx.
14122	 */
14123	mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex);
14124
14125	__perf_pmu_remove(src_ctx, src_cpu, pmu, &src_ctx->pinned_groups, &events);
14126	__perf_pmu_remove(src_ctx, src_cpu, pmu, &src_ctx->flexible_groups, &events);
14127
14128	if (!list_empty(&events)) {
14129		/*
14130		 * Wait for the events to quiesce before re-instating them.
14131		 */
14132		synchronize_rcu();
14133
14134		__perf_pmu_install(dst_ctx, dst_cpu, pmu, &events);
14135	}
14136
14137	mutex_unlock(&dst_ctx->mutex);
14138	mutex_unlock(&src_ctx->mutex);
14139}
14140EXPORT_SYMBOL_GPL(perf_pmu_migrate_context);
14141
14142static void sync_child_event(struct perf_event *child_event,
14143			     struct task_struct *task)
14144{
14145	struct perf_event *parent_event = child_event->parent;
14146	u64 child_val;
14147
14148	if (child_event->attr.inherit_stat) {
14149		if (task && task != TASK_TOMBSTONE)
14150			perf_event_read_event(child_event, task);
14151	}
14152
14153	child_val = perf_event_count(child_event, false);
14154
14155	/*
14156	 * Add back the child's count to the parent's count:
14157	 */
14158	atomic64_add(child_val, &parent_event->child_count);
14159	atomic64_add(child_event->total_time_enabled,
14160		     &parent_event->child_total_time_enabled);
14161	atomic64_add(child_event->total_time_running,
14162		     &parent_event->child_total_time_running);
14163}
14164
14165static void
14166perf_event_exit_event(struct perf_event *event,
14167		      struct perf_event_context *ctx,
14168		      struct task_struct *task,
14169		      bool revoke)
14170{
14171	struct perf_event *parent_event = event->parent;
14172	unsigned long detach_flags = DETACH_EXIT;
14173	unsigned int attach_state;
14174
14175	if (parent_event) {
14176		/*
14177		 * Do not destroy the 'original' grouping; because of the
14178		 * context switch optimization the original events could've
14179		 * ended up in a random child task.
14180		 *
14181		 * If we were to destroy the original group, all group related
14182		 * operations would cease to function properly after this
14183		 * random child dies.
14184		 *
14185		 * Do destroy all inherited groups, we don't care about those
14186		 * and being thorough is better.
14187		 */
14188		detach_flags |= DETACH_GROUP | DETACH_CHILD;
14189		mutex_lock(&parent_event->child_mutex);
14190		/* PERF_ATTACH_ITRACE might be set concurrently */
14191		attach_state = READ_ONCE(event->attach_state);
14192
14193		if (attach_state & PERF_ATTACH_CHILD)
14194			sync_child_event(event, task);
14195	}
14196
14197	if (revoke)
14198		detach_flags |= DETACH_GROUP | DETACH_REVOKE;
14199
14200	perf_remove_from_context(event, detach_flags);
14201	/*
14202	 * Child events can be freed.
14203	 */
14204	if (parent_event) {
14205		mutex_unlock(&parent_event->child_mutex);
14206
14207		/*
14208		 * Match the refcount initialization. Make sure it doesn't happen
14209		 * twice if pmu_detach_event() calls it on an already exited task.
14210		 */
14211		if (attach_state & PERF_ATTACH_CHILD) {
14212			/*
14213			 * Kick perf_poll() for is_event_hup();
14214			 */
14215			perf_event_wakeup(parent_event);
14216			/*
14217			 * pmu_detach_event() will have an extra refcount.
14218			 * perf_pending_task() might have one too.
14219			 */
14220			put_event(event);
14221		}
14222
14223		return;
14224	}
14225
14226	/*
14227	 * Parent events are governed by their filedesc, retain them.
14228	 */
14229	perf_event_wakeup(event);
14230}
14231
14232static void perf_event_exit_task_context(struct task_struct *task, bool exit)
14233{
14234	struct perf_event_context *ctx, *clone_ctx = NULL;
14235	struct perf_event *child_event, *next;
14236
14237	ctx = perf_pin_task_context(task);
14238	if (!ctx)
14239		return;
14240
14241	/*
14242	 * In order to reduce the amount of tricky in ctx tear-down, we hold
14243	 * ctx::mutex over the entire thing. This serializes against almost
14244	 * everything that wants to access the ctx.
14245	 *
14246	 * The exception is sys_perf_event_open() /
14247	 * perf_event_create_kernel_count() which does find_get_context()
14248	 * without ctx::mutex (it cannot because of the move_group double mutex
14249	 * lock thing). See the comments in perf_install_in_context().
14250	 */
14251	mutex_lock(&ctx->mutex);
14252
14253	/*
14254	 * In a single ctx::lock section, de-schedule the events and detach the
14255	 * context from the task such that we cannot ever get it scheduled back
14256	 * in.
14257	 */
14258	raw_spin_lock_irq(&ctx->lock);
14259	if (exit)
14260		task_ctx_sched_out(ctx, NULL, EVENT_ALL);
14261
14262	/*
14263	 * Now that the context is inactive, destroy the task <-> ctx relation
14264	 * and mark the context dead.
14265	 */
14266	RCU_INIT_POINTER(task->perf_event_ctxp, NULL);
14267	put_ctx(ctx); /* cannot be last */
14268	WRITE_ONCE(ctx->task, TASK_TOMBSTONE);
14269	put_task_struct(task); /* cannot be last */
14270
14271	clone_ctx = unclone_ctx(ctx);
14272	raw_spin_unlock_irq(&ctx->lock);
14273
14274	if (clone_ctx)
14275		put_ctx(clone_ctx);
14276
14277	/*
14278	 * Report the task dead after unscheduling the events so that we
14279	 * won't get any samples after PERF_RECORD_EXIT. We can however still
14280	 * get a few PERF_RECORD_READ events.
14281	 */
14282	if (exit)
14283		perf_event_task(task, ctx, 0);
14284
14285	list_for_each_entry_safe(child_event, next, &ctx->event_list, event_entry)
14286		perf_event_exit_event(child_event, ctx, exit ? task : NULL, false);
14287
14288	mutex_unlock(&ctx->mutex);
14289
14290	if (!exit) {
14291		/*
14292		 * perf_event_release_kernel() could still have a reference on
14293		 * this context. In that case we must wait for these events to
14294		 * have been freed (in particular all their references to this
14295		 * task must've been dropped).
14296		 *
14297		 * Without this copy_process() will unconditionally free this
14298		 * task (irrespective of its reference count) and
14299		 * _free_event()'s put_task_struct(event->hw.target) will be a
14300		 * use-after-free.
14301		 *
14302		 * Wait for all events to drop their context reference.
14303		 */
14304		wait_var_event(&ctx->refcount,
14305			       refcount_read(&ctx->refcount) == 1);
14306	}
14307	put_ctx(ctx);
14308}
14309
14310/*
14311 * When a task exits, feed back event values to parent events.
14312 *
14313 * Can be called with exec_update_lock held when called from
14314 * setup_new_exec().
14315 */
14316void perf_event_exit_task(struct task_struct *task)
14317{
14318	struct perf_event *event, *tmp;
14319
14320	WARN_ON_ONCE(task != current);
14321
14322	mutex_lock(&task->perf_event_mutex);
14323	list_for_each_entry_safe(event, tmp, &task->perf_event_list,
14324				 owner_entry) {
14325		list_del_init(&event->owner_entry);
14326
14327		/*
14328		 * Ensure the list deletion is visible before we clear
14329		 * the owner, closes a race against perf_release() where
14330		 * we need to serialize on the owner->perf_event_mutex.
14331		 */
14332		smp_store_release(&event->owner, NULL);
14333	}
14334	mutex_unlock(&task->perf_event_mutex);
14335
14336	perf_event_exit_task_context(task, true);
14337
14338	/*
14339	 * The perf_event_exit_task_context calls perf_event_task
14340	 * with task's task_ctx, which generates EXIT events for
14341	 * task contexts and sets task->perf_event_ctxp[] to NULL.
14342	 * At this point we need to send EXIT events to cpu contexts.
14343	 */
14344	perf_event_task(task, NULL, 0);
14345
14346	/*
14347	 * Detach the perf_ctx_data for the system-wide event.
14348	 *
14349	 * Done without holding global_ctx_data_rwsem; typically
14350	 * attach_global_ctx_data() will skip over this task, but otherwise
14351	 * attach_task_ctx_data() will observe PF_EXITING.
14352	 */
14353	detach_task_ctx_data(task);
14354}
14355
14356/*
14357 * Free a context as created by inheritance by perf_event_init_task() below,
14358 * used by fork() in case of fail.
14359 *
14360 * Even though the task has never lived, the context and events have been
14361 * exposed through the child_list, so we must take care tearing it all down.
14362 */
14363void perf_event_free_task(struct task_struct *task)
14364{
14365	perf_event_exit_task_context(task, false);
14366}
14367
14368void perf_event_delayed_put(struct task_struct *task)
14369{
14370	WARN_ON_ONCE(task->perf_event_ctxp);
14371}
14372
14373struct file *perf_event_get(unsigned int fd)
14374{
14375	struct file *file = fget(fd);
14376	if (!file)
14377		return ERR_PTR(-EBADF);
14378
14379	if (file->f_op != &perf_fops) {
14380		fput(file);
14381		return ERR_PTR(-EBADF);
14382	}
14383
14384	return file;
14385}
14386
14387const struct perf_event *perf_get_event(struct file *file)
14388{
14389	if (file->f_op != &perf_fops)
14390		return ERR_PTR(-EINVAL);
14391
14392	return file->private_data;
14393}
14394
14395const struct perf_event_attr *perf_event_attrs(struct perf_event *event)
14396{
14397	if (!event)
14398		return ERR_PTR(-EINVAL);
14399
14400	return &event->attr;
14401}
14402
14403int perf_allow_kernel(void)
14404{
14405	if (sysctl_perf_event_paranoid > 1 && !perfmon_capable())
14406		return -EACCES;
14407
14408	return security_perf_event_open(PERF_SECURITY_KERNEL);
14409}
14410EXPORT_SYMBOL_GPL(perf_allow_kernel);
14411
14412/*
14413 * Inherit an event from parent task to child task.
14414 *
14415 * Returns:
14416 *  - valid pointer on success
14417 *  - NULL for orphaned events
14418 *  - IS_ERR() on error
14419 */
14420static struct perf_event *
14421inherit_event(struct perf_event *parent_event,
14422	      struct task_struct *parent,
14423	      struct perf_event_context *parent_ctx,
14424	      struct task_struct *child,
14425	      struct perf_event *group_leader,
14426	      struct perf_event_context *child_ctx)
14427{
14428	enum perf_event_state parent_state = parent_event->state;
14429	struct perf_event_pmu_context *pmu_ctx;
14430	struct perf_event *child_event;
14431	unsigned long flags;
14432
14433	/*
14434	 * Instead of creating recursive hierarchies of events,
14435	 * we link inherited events back to the original parent,
14436	 * which has a filp for sure, which we use as the reference
14437	 * count:
14438	 */
14439	if (parent_event->parent)
14440		parent_event = parent_event->parent;
14441
14442	if (parent_event->state <= PERF_EVENT_STATE_REVOKED)
14443		return NULL;
14444
14445	/*
14446	 * Event creation should be under SRCU, see perf_pmu_unregister().
14447	 */
14448	guard(srcu)(&pmus_srcu);
14449
14450	child_event = perf_event_alloc(&parent_event->attr,
14451					   parent_event->cpu,
14452					   child,
14453					   group_leader, parent_event,
14454					   NULL, NULL, -1);
14455	if (IS_ERR(child_event))
14456		return child_event;
14457
14458	get_ctx(child_ctx);
14459	child_event->ctx = child_ctx;
14460
14461	pmu_ctx = find_get_pmu_context(parent_event->pmu_ctx->pmu, child_ctx, child_event);
14462	if (IS_ERR(pmu_ctx)) {
14463		free_event(child_event);
14464		return ERR_CAST(pmu_ctx);
14465	}
14466	child_event->pmu_ctx = pmu_ctx;
14467
14468	/*
14469	 * is_orphaned_event() and list_add_tail(&parent_event->child_list)
14470	 * must be under the same lock in order to serialize against
14471	 * perf_event_release_kernel(), such that either we must observe
14472	 * is_orphaned_event() or they will observe us on the child_list.
14473	 */
14474	mutex_lock(&parent_event->child_mutex);
14475	if (is_orphaned_event(parent_event) ||
14476	    !atomic_long_inc_not_zero(&parent_event->refcount)) {
14477		mutex_unlock(&parent_event->child_mutex);
14478		free_event(child_event);
14479		return NULL;
14480	}
14481
14482	/*
14483	 * Make the child state follow the state of the parent event,
14484	 * not its attr.disabled bit.  We hold the parent's mutex,
14485	 * so we won't race with perf_event_{en, dis}able_family.
14486	 */
14487	if (parent_state >= PERF_EVENT_STATE_INACTIVE)
14488		child_event->state = PERF_EVENT_STATE_INACTIVE;
14489	else
14490		child_event->state = PERF_EVENT_STATE_OFF;
14491
14492	if (parent_event->attr.freq) {
14493		u64 sample_period = parent_event->hw.sample_period;
14494		struct hw_perf_event *hwc = &child_event->hw;
14495
14496		hwc->sample_period = sample_period;
14497		hwc->last_period   = sample_period;
14498
14499		local64_set(&hwc->period_left, sample_period);
14500	}
14501
14502	child_event->overflow_handler = parent_event->overflow_handler;
14503	child_event->overflow_handler_context
14504		= parent_event->overflow_handler_context;
14505
14506	/*
14507	 * Precalculate sample_data sizes
14508	 */
14509	perf_event__header_size(child_event);
14510	perf_event__id_header_size(child_event);
14511
14512	/*
14513	 * Link it up in the child's context:
14514	 */
14515	raw_spin_lock_irqsave(&child_ctx->lock, flags);
14516	add_event_to_ctx(child_event, child_ctx);
14517	child_event->attach_state |= PERF_ATTACH_CHILD;
14518	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
14519
14520	/*
14521	 * Link this into the parent event's child list
14522	 */
14523	list_add_tail(&child_event->child_list, &parent_event->child_list);
14524	mutex_unlock(&parent_event->child_mutex);
14525
14526	return child_event;
14527}
14528
14529/*
14530 * Inherits an event group.
14531 *
14532 * This will quietly suppress orphaned events; !inherit_event() is not an error.
14533 * This matches with perf_event_release_kernel() removing all child events.
14534 *
14535 * Returns:
14536 *  - 0 on success
14537 *  - <0 on error
14538 */
14539static int inherit_group(struct perf_event *parent_event,
14540	      struct task_struct *parent,
14541	      struct perf_event_context *parent_ctx,
14542	      struct task_struct *child,
14543	      struct perf_event_context *child_ctx)
14544{
14545	struct perf_event *leader;
14546	struct perf_event *sub;
14547	struct perf_event *child_ctr;
14548
14549	leader = inherit_event(parent_event, parent, parent_ctx,
14550				 child, NULL, child_ctx);
14551	if (IS_ERR(leader))
14552		return PTR_ERR(leader);
14553	/*
14554	 * @leader can be NULL here because of is_orphaned_event(). In this
14555	 * case inherit_event() will create individual events, similar to what
14556	 * perf_group_detach() would do anyway.
14557	 */
14558	for_each_sibling_event(sub, parent_event) {
14559		child_ctr = inherit_event(sub, parent, parent_ctx,
14560					    child, leader, child_ctx);
14561		if (IS_ERR(child_ctr))
14562			return PTR_ERR(child_ctr);
14563
14564		if (sub->aux_event == parent_event && child_ctr &&
14565		    !perf_get_aux_event(child_ctr, leader))
14566			return -EINVAL;
14567	}
14568	if (leader)
14569		leader->group_generation = parent_event->group_generation;
14570	return 0;
14571}
14572
14573/*
14574 * Creates the child task context and tries to inherit the event-group.
14575 *
14576 * Clears @inherited_all on !attr.inherited or error. Note that we'll leave
14577 * inherited_all set when we 'fail' to inherit an orphaned event; this is
14578 * consistent with perf_event_release_kernel() removing all child events.
14579 *
14580 * Returns:
14581 *  - 0 on success
14582 *  - <0 on error
14583 */
14584static int
14585inherit_task_group(struct perf_event *event, struct task_struct *parent,
14586		   struct perf_event_context *parent_ctx,
14587		   struct task_struct *child,
14588		   u64 clone_flags, int *inherited_all)
14589{
14590	struct perf_event_context *child_ctx;
14591	int ret;
14592
14593	if (!event->attr.inherit ||
14594	    (event->attr.inherit_thread && !(clone_flags & CLONE_THREAD)) ||
14595	    /* Do not inherit if sigtrap and signal handlers were cleared. */
14596	    (event->attr.sigtrap && (clone_flags & CLONE_CLEAR_SIGHAND))) {
14597		*inherited_all = 0;
14598		return 0;
14599	}
14600
14601	child_ctx = child->perf_event_ctxp;
14602	if (!child_ctx) {
14603		/*
14604		 * This is executed from the parent task context, so
14605		 * inherit events that have been marked for cloning.
14606		 * First allocate and initialize a context for the
14607		 * child.
14608		 */
14609		child_ctx = alloc_perf_context(child);
14610		if (!child_ctx)
14611			return -ENOMEM;
14612
14613		child->perf_event_ctxp = child_ctx;
14614	}
14615
14616	ret = inherit_group(event, parent, parent_ctx, child, child_ctx);
14617	if (ret)
14618		*inherited_all = 0;
14619
14620	return ret;
14621}
14622
14623/*
14624 * Initialize the perf_event context in task_struct
14625 */
14626static int perf_event_init_context(struct task_struct *child, u64 clone_flags)
14627{
14628	struct perf_event_context *child_ctx, *parent_ctx;
14629	struct perf_event_context *cloned_ctx;
14630	struct perf_event *event;
14631	struct task_struct *parent = current;
14632	int inherited_all = 1;
14633	unsigned long flags;
14634	int ret = 0;
14635
14636	if (likely(!parent->perf_event_ctxp))
14637		return 0;
14638
14639	/*
14640	 * If the parent's context is a clone, pin it so it won't get
14641	 * swapped under us.
14642	 */
14643	parent_ctx = perf_pin_task_context(parent);
14644	if (!parent_ctx)
14645		return 0;
14646
14647	/*
14648	 * No need to check if parent_ctx != NULL here; since we saw
14649	 * it non-NULL earlier, the only reason for it to become NULL
14650	 * is if we exit, and since we're currently in the middle of
14651	 * a fork we can't be exiting at the same time.
14652	 */
14653
14654	/*
14655	 * Lock the parent list. No need to lock the child - not PID
14656	 * hashed yet and not running, so nobody can access it.
14657	 */
14658	mutex_lock(&parent_ctx->mutex);
14659
14660	/*
14661	 * We dont have to disable NMIs - we are only looking at
14662	 * the list, not manipulating it:
14663	 */
14664	perf_event_groups_for_each(event, &parent_ctx->pinned_groups) {
14665		ret = inherit_task_group(event, parent, parent_ctx,
14666					 child, clone_flags, &inherited_all);
14667		if (ret)
14668			goto out_unlock;
14669	}
14670
14671	/*
14672	 * We can't hold ctx->lock when iterating the ->flexible_group list due
14673	 * to allocations, but we need to prevent rotation because
14674	 * rotate_ctx() will change the list from interrupt context.
14675	 */
14676	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
14677	parent_ctx->rotate_disable = 1;
14678	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
14679
14680	perf_event_groups_for_each(event, &parent_ctx->flexible_groups) {
14681		ret = inherit_task_group(event, parent, parent_ctx,
14682					 child, clone_flags, &inherited_all);
14683		if (ret)
14684			goto out_unlock;
14685	}
14686
14687	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
14688	parent_ctx->rotate_disable = 0;
14689
14690	child_ctx = child->perf_event_ctxp;
14691
14692	if (child_ctx && inherited_all) {
14693		/*
14694		 * Mark the child context as a clone of the parent
14695		 * context, or of whatever the parent is a clone of.
14696		 *
14697		 * Note that if the parent is a clone, the holding of
14698		 * parent_ctx->lock avoids it from being uncloned.
14699		 */
14700		cloned_ctx = parent_ctx->parent_ctx;
14701		if (cloned_ctx) {
14702			child_ctx->parent_ctx = cloned_ctx;
14703			child_ctx->parent_gen = parent_ctx->parent_gen;
14704		} else {
14705			child_ctx->parent_ctx = parent_ctx;
14706			child_ctx->parent_gen = parent_ctx->generation;
14707		}
14708		get_ctx(child_ctx->parent_ctx);
14709	}
14710
14711	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
14712out_unlock:
14713	mutex_unlock(&parent_ctx->mutex);
14714
14715	perf_unpin_context(parent_ctx);
14716	put_ctx(parent_ctx);
14717
14718	return ret;
14719}
14720
14721/*
14722 * Initialize the perf_event context in task_struct
14723 */
14724int perf_event_init_task(struct task_struct *child, u64 clone_flags)
14725{
14726	int ret;
14727
14728	memset(child->perf_recursion, 0, sizeof(child->perf_recursion));
14729	child->perf_event_ctxp = NULL;
14730	mutex_init(&child->perf_event_mutex);
14731	INIT_LIST_HEAD(&child->perf_event_list);
14732	child->perf_ctx_data = NULL;
14733
14734	ret = perf_event_init_context(child, clone_flags);
14735	if (ret) {
14736		perf_event_free_task(child);
14737		return ret;
14738	}
14739
14740	return 0;
14741}
14742
14743static void __init perf_event_init_all_cpus(void)
14744{
14745	struct swevent_htable *swhash;
14746	struct perf_cpu_context *cpuctx;
14747	int cpu;
14748
14749	zalloc_cpumask_var(&perf_online_mask, GFP_KERNEL);
14750	zalloc_cpumask_var(&perf_online_core_mask, GFP_KERNEL);
14751	zalloc_cpumask_var(&perf_online_die_mask, GFP_KERNEL);
14752	zalloc_cpumask_var(&perf_online_cluster_mask, GFP_KERNEL);
14753	zalloc_cpumask_var(&perf_online_pkg_mask, GFP_KERNEL);
14754	zalloc_cpumask_var(&perf_online_sys_mask, GFP_KERNEL);
14755
14756
14757	for_each_possible_cpu(cpu) {
14758		swhash = &per_cpu(swevent_htable, cpu);
14759		mutex_init(&swhash->hlist_mutex);
14760
14761		INIT_LIST_HEAD(&per_cpu(pmu_sb_events.list, cpu));
14762		raw_spin_lock_init(&per_cpu(pmu_sb_events.lock, cpu));
14763
14764		INIT_LIST_HEAD(&per_cpu(sched_cb_list, cpu));
14765
14766		cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
14767		__perf_event_init_context(&cpuctx->ctx);
14768		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
14769		lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
14770		cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask);
14771		cpuctx->heap_size = ARRAY_SIZE(cpuctx->heap_default);
14772		cpuctx->heap = cpuctx->heap_default;
14773	}
14774}
14775
14776static void perf_swevent_init_cpu(unsigned int cpu)
14777{
14778	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
14779
14780	mutex_lock(&swhash->hlist_mutex);
14781	if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) {
14782		struct swevent_hlist *hlist;
14783
14784		hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
14785		WARN_ON(!hlist);
14786		rcu_assign_pointer(swhash->swevent_hlist, hlist);
14787	}
14788	mutex_unlock(&swhash->hlist_mutex);
14789}
14790
14791#if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC_CORE
14792static void __perf_event_exit_context(void *__info)
14793{
14794	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
14795	struct perf_event_context *ctx = __info;
14796	struct perf_event *event;
14797
14798	raw_spin_lock(&ctx->lock);
14799	ctx_sched_out(ctx, NULL, EVENT_TIME);
14800	list_for_each_entry(event, &ctx->event_list, event_entry)
14801		__perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP);
14802	raw_spin_unlock(&ctx->lock);
14803}
14804
14805static void perf_event_clear_cpumask(unsigned int cpu)
14806{
14807	int target[PERF_PMU_MAX_SCOPE];
14808	unsigned int scope;
14809	struct pmu *pmu;
14810
14811	cpumask_clear_cpu(cpu, perf_online_mask);
14812
14813	for (scope = PERF_PMU_SCOPE_NONE + 1; scope < PERF_PMU_MAX_SCOPE; scope++) {
14814		const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(scope, cpu);
14815		struct cpumask *pmu_cpumask = perf_scope_cpumask(scope);
14816
14817		target[scope] = -1;
14818		if (WARN_ON_ONCE(!pmu_cpumask || !cpumask))
14819			continue;
14820
14821		if (!cpumask_test_and_clear_cpu(cpu, pmu_cpumask))
14822			continue;
14823		target[scope] = cpumask_any_but(cpumask, cpu);
14824		if (target[scope] < nr_cpu_ids)
14825			cpumask_set_cpu(target[scope], pmu_cpumask);
14826	}
14827
14828	/* migrate */
14829	list_for_each_entry(pmu, &pmus, entry) {
14830		if (pmu->scope == PERF_PMU_SCOPE_NONE ||
14831		    WARN_ON_ONCE(pmu->scope >= PERF_PMU_MAX_SCOPE))
14832			continue;
14833
14834		if (target[pmu->scope] >= 0 && target[pmu->scope] < nr_cpu_ids)
14835			perf_pmu_migrate_context(pmu, cpu, target[pmu->scope]);
14836	}
14837}
14838
14839static void perf_event_exit_cpu_context(int cpu)
14840{
14841	struct perf_cpu_context *cpuctx;
14842	struct perf_event_context *ctx;
14843
14844	// XXX simplify cpuctx->online
14845	mutex_lock(&pmus_lock);
14846	/*
14847	 * Clear the cpumasks, and migrate to other CPUs if possible.
14848	 * Must be invoked before the __perf_event_exit_context.
14849	 */
14850	perf_event_clear_cpumask(cpu);
14851	cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
14852	ctx = &cpuctx->ctx;
14853
14854	mutex_lock(&ctx->mutex);
14855	smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
14856	cpuctx->online = 0;
14857	mutex_unlock(&ctx->mutex);
14858	mutex_unlock(&pmus_lock);
14859}
14860#else
14861
14862static void perf_event_exit_cpu_context(int cpu) { }
14863
14864#endif
14865
14866static void perf_event_setup_cpumask(unsigned int cpu)
14867{
14868	struct cpumask *pmu_cpumask;
14869	unsigned int scope;
14870
14871	/*
14872	 * Early boot stage, the cpumask hasn't been set yet.
14873	 * The perf_online_<domain>_masks includes the first CPU of each domain.
14874	 * Always unconditionally set the boot CPU for the perf_online_<domain>_masks.
14875	 */
14876	if (cpumask_empty(perf_online_mask)) {
14877		for (scope = PERF_PMU_SCOPE_NONE + 1; scope < PERF_PMU_MAX_SCOPE; scope++) {
14878			pmu_cpumask = perf_scope_cpumask(scope);
14879			if (WARN_ON_ONCE(!pmu_cpumask))
14880				continue;
14881			cpumask_set_cpu(cpu, pmu_cpumask);
14882		}
14883		goto end;
14884	}
14885
14886	for (scope = PERF_PMU_SCOPE_NONE + 1; scope < PERF_PMU_MAX_SCOPE; scope++) {
14887		const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(scope, cpu);
14888
14889		pmu_cpumask = perf_scope_cpumask(scope);
14890
14891		if (WARN_ON_ONCE(!pmu_cpumask || !cpumask))
14892			continue;
14893
14894		if (!cpumask_empty(cpumask) &&
14895		    cpumask_any_and(pmu_cpumask, cpumask) >= nr_cpu_ids)
14896			cpumask_set_cpu(cpu, pmu_cpumask);
14897	}
14898end:
14899	cpumask_set_cpu(cpu, perf_online_mask);
14900}
14901
14902int perf_event_init_cpu(unsigned int cpu)
14903{
14904	struct perf_cpu_context *cpuctx;
14905	struct perf_event_context *ctx;
14906
14907	perf_swevent_init_cpu(cpu);
14908
14909	mutex_lock(&pmus_lock);
14910	perf_event_setup_cpumask(cpu);
14911	cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
14912	ctx = &cpuctx->ctx;
14913
14914	mutex_lock(&ctx->mutex);
14915	cpuctx->online = 1;
14916	mutex_unlock(&ctx->mutex);
14917	mutex_unlock(&pmus_lock);
14918
14919	return 0;
14920}
14921
14922int perf_event_exit_cpu(unsigned int cpu)
14923{
14924	perf_event_exit_cpu_context(cpu);
14925	return 0;
14926}
14927
14928static int
14929perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
14930{
14931	int cpu;
14932
14933	for_each_online_cpu(cpu)
14934		perf_event_exit_cpu(cpu);
14935
14936	return NOTIFY_OK;
14937}
14938
14939/*
14940 * Run the perf reboot notifier at the very last possible moment so that
14941 * the generic watchdog code runs as long as possible.
14942 */
14943static struct notifier_block perf_reboot_notifier = {
14944	.notifier_call = perf_reboot,
14945	.priority = INT_MIN,
14946};
14947
14948void __init perf_event_init(void)
14949{
14950	int ret;
14951
14952	idr_init(&pmu_idr);
14953
14954	unwind_deferred_init(&perf_unwind_work,
14955			     perf_unwind_deferred_callback);
14956
14957	perf_event_init_all_cpus();
14958	init_srcu_struct(&pmus_srcu);
14959	perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
14960	perf_pmu_register(&perf_cpu_clock, "cpu_clock", -1);
14961	perf_pmu_register(&perf_task_clock, "task_clock", -1);
14962	perf_tp_register();
14963	perf_event_init_cpu(smp_processor_id());
14964	register_reboot_notifier(&perf_reboot_notifier);
14965
14966	ret = init_hw_breakpoint();
14967	WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
14968
14969	perf_event_cache = KMEM_CACHE(perf_event, SLAB_PANIC);
14970
14971	/*
14972	 * Build time assertion that we keep the data_head at the intended
14973	 * location.  IOW, validation we got the __reserved[] size right.
14974	 */
14975	BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
14976		     != 1024);
14977}
14978
14979ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr,
14980			      char *page)
14981{
14982	struct perf_pmu_events_attr *pmu_attr =
14983		container_of(attr, struct perf_pmu_events_attr, attr);
14984
14985	if (pmu_attr->event_str)
14986		return sprintf(page, "%s\n", pmu_attr->event_str);
14987
14988	return 0;
14989}
14990EXPORT_SYMBOL_GPL(perf_event_sysfs_show);
14991
14992static int __init perf_event_sysfs_init(void)
14993{
14994	struct pmu *pmu;
14995	int ret;
14996
14997	mutex_lock(&pmus_lock);
14998
14999	ret = bus_register(&pmu_bus);
15000	if (ret)
15001		goto unlock;
15002
15003	list_for_each_entry(pmu, &pmus, entry) {
15004		if (pmu->dev)
15005			continue;
15006
15007		ret = pmu_dev_alloc(pmu);
15008		WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
15009	}
15010	pmu_bus_running = 1;
15011	ret = 0;
15012
15013unlock:
15014	mutex_unlock(&pmus_lock);
15015
15016	return ret;
15017}
15018device_initcall(perf_event_sysfs_init);
15019
15020#ifdef CONFIG_CGROUP_PERF
15021static struct cgroup_subsys_state *
15022perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
15023{
15024	struct perf_cgroup *jc;
15025
15026	jc = kzalloc(sizeof(*jc), GFP_KERNEL);
15027	if (!jc)
15028		return ERR_PTR(-ENOMEM);
15029
15030	jc->info = alloc_percpu(struct perf_cgroup_info);
15031	if (!jc->info) {
15032		kfree(jc);
15033		return ERR_PTR(-ENOMEM);
15034	}
15035
15036	return &jc->css;
15037}
15038
15039static void perf_cgroup_css_free(struct cgroup_subsys_state *css)
15040{
15041	struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css);
15042
15043	free_percpu(jc->info);
15044	kfree(jc);
15045}
15046
15047static int perf_cgroup_css_online(struct cgroup_subsys_state *css)
15048{
15049	perf_event_cgroup(css->cgroup);
15050	return 0;
15051}
15052
15053static int __perf_cgroup_move(void *info)
15054{
15055	struct task_struct *task = info;
15056
15057	preempt_disable();
15058	perf_cgroup_switch(task);
15059	preempt_enable();
15060
15061	return 0;
15062}
15063
15064static void perf_cgroup_attach(struct cgroup_taskset *tset)
15065{
15066	struct task_struct *task;
15067	struct cgroup_subsys_state *css;
15068
15069	cgroup_taskset_for_each(task, css, tset)
15070		task_function_call(task, __perf_cgroup_move, task);
15071}
15072
15073struct cgroup_subsys perf_event_cgrp_subsys = {
15074	.css_alloc	= perf_cgroup_css_alloc,
15075	.css_free	= perf_cgroup_css_free,
15076	.css_online	= perf_cgroup_css_online,
15077	.attach		= perf_cgroup_attach,
15078	/*
15079	 * Implicitly enable on dfl hierarchy so that perf events can
15080	 * always be filtered by cgroup2 path as long as perf_event
15081	 * controller is not mounted on a legacy hierarchy.
15082	 */
15083	.implicit_on_dfl = true,
15084	.threaded	= true,
15085};
15086#endif /* CONFIG_CGROUP_PERF */
15087
15088DEFINE_STATIC_CALL_RET0(perf_snapshot_branch_stack, perf_snapshot_branch_stack_t);