posted_intr.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef _X86_POSTED_INTR_H
  3#define _X86_POSTED_INTR_H
  4
  5#include <asm/cmpxchg.h>
  6#include <asm/rwonce.h>
  7#include <asm/irq_vectors.h>
  8
  9#include <linux/bitmap.h>
 10
 11#define POSTED_INTR_ON  0
 12#define POSTED_INTR_SN  1
 13
 14#define PID_TABLE_ENTRY_VALID 1
 15
 16#define NR_PIR_VECTORS	256
 17#define NR_PIR_WORDS	(NR_PIR_VECTORS / BITS_PER_LONG)
 18
 19/* Posted-Interrupt Descriptor */
 20struct pi_desc {
 21	unsigned long pir[NR_PIR_WORDS];     /* Posted interrupt requested */
 22	union {
 23		struct {
 24			u16	notifications; /* Suppress and outstanding bits */
 25			u8	nv;
 26			u8	rsvd_2;
 27			u32	ndst;
 28		};
 29		u64 control;
 30	};
 31	u32 rsvd[6];
 32} __aligned(64);
 33
 34/*
 35 * De-multiplexing posted interrupts is on the performance path, the code
 36 * below is written to optimize the cache performance based on the following
 37 * considerations:
 38 * 1.Posted interrupt descriptor (PID) fits in a cache line that is frequently
 39 *   accessed by both CPU and IOMMU.
 40 * 2.During software processing of posted interrupts, the CPU needs to do
 41 *   natural width read and xchg for checking and clearing posted interrupt
 42 *   request (PIR), a 256 bit field within the PID.
 43 * 3.On the other side, the IOMMU does atomic swaps of the entire PID cache
 44 *   line when posting interrupts and setting control bits.
 45 * 4.The CPU can access the cache line a magnitude faster than the IOMMU.
 46 * 5.Each time the IOMMU does interrupt posting to the PIR will evict the PID
 47 *   cache line. The cache line states after each operation are as follows,
 48 *   assuming a 64-bit kernel:
 49 *   CPU		IOMMU			PID Cache line state
 50 *   ---------------------------------------------------------------
 51 *...read64					exclusive
 52 *...lock xchg64				modified
 53 *...			post/atomic swap	invalid
 54 *...-------------------------------------------------------------
 55 *
 56 * To reduce L1 data cache miss, it is important to avoid contention with
 57 * IOMMU's interrupt posting/atomic swap. Therefore, a copy of PIR is used
 58 * when processing posted interrupts in software, e.g. to dispatch interrupt
 59 * handlers for posted MSIs, or to move interrupts from the PIR to the vIRR
 60 * in KVM.
 61 *
 62 * In addition, the code is trying to keep the cache line state consistent
 63 * as much as possible. e.g. when making a copy and clearing the PIR
 64 * (assuming non-zero PIR bits are present in the entire PIR), it does:
 65 *		read, read, read, read, xchg, xchg, xchg, xchg
 66 * instead of:
 67 *		read, xchg, read, xchg, read, xchg, read, xchg
 68 */
 69static __always_inline bool pi_harvest_pir(unsigned long *pir,
 70					   unsigned long *pir_vals)
 71{
 72	unsigned long pending = 0;
 73	int i;
 74
 75	for (i = 0; i < NR_PIR_WORDS; i++) {
 76		pir_vals[i] = READ_ONCE(pir[i]);
 77		pending |= pir_vals[i];
 78	}
 79
 80	if (!pending)
 81		return false;
 82
 83	for (i = 0; i < NR_PIR_WORDS; i++) {
 84		if (!pir_vals[i])
 85			continue;
 86
 87		pir_vals[i] = arch_xchg(&pir[i], 0);
 88	}
 89
 90	return true;
 91}
 92
 93static inline bool pi_test_and_set_on(struct pi_desc *pi_desc)
 94{
 95	return test_and_set_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control);
 96}
 97
 98static inline bool pi_test_and_clear_on(struct pi_desc *pi_desc)
 99{
100	return test_and_clear_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control);
101}
102
103static inline bool pi_test_and_clear_sn(struct pi_desc *pi_desc)
104{
105	return test_and_clear_bit(POSTED_INTR_SN, (unsigned long *)&pi_desc->control);
106}
107
108static inline bool pi_test_and_set_pir(int vector, struct pi_desc *pi_desc)
109{
110	return test_and_set_bit(vector, pi_desc->pir);
111}
112
113static inline bool pi_is_pir_empty(struct pi_desc *pi_desc)
114{
115	return bitmap_empty(pi_desc->pir, NR_VECTORS);
116}
117
118static inline void pi_set_sn(struct pi_desc *pi_desc)
119{
120	set_bit(POSTED_INTR_SN, (unsigned long *)&pi_desc->control);
121}
122
123static inline void pi_set_on(struct pi_desc *pi_desc)
124{
125	set_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control);
126}
127
128static inline void pi_clear_on(struct pi_desc *pi_desc)
129{
130	clear_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control);
131}
132
133static inline void pi_clear_sn(struct pi_desc *pi_desc)
134{
135	clear_bit(POSTED_INTR_SN, (unsigned long *)&pi_desc->control);
136}
137
138static inline bool pi_test_on(struct pi_desc *pi_desc)
139{
140	return test_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control);
141}
142
143static inline bool pi_test_sn(struct pi_desc *pi_desc)
144{
145	return test_bit(POSTED_INTR_SN, (unsigned long *)&pi_desc->control);
146}
147
148static inline bool pi_test_pir(int vector, struct pi_desc *pi_desc)
149{
150	return test_bit(vector, (unsigned long *)pi_desc->pir);
151}
152
153/* Non-atomic helpers */
154static inline void __pi_set_sn(struct pi_desc *pi_desc)
155{
156	pi_desc->notifications |= BIT(POSTED_INTR_SN);
157}
158
159static inline void __pi_clear_sn(struct pi_desc *pi_desc)
160{
161	pi_desc->notifications &= ~BIT(POSTED_INTR_SN);
162}
163
164#ifdef CONFIG_X86_POSTED_MSI
165/*
166 * Not all external vectors are subject to interrupt remapping, e.g. IOMMU's
167 * own interrupts. Here we do not distinguish them since those vector bits in
168 * PIR will always be zero.
169 */
170static inline bool pi_pending_this_cpu(unsigned int vector)
171{
172	struct pi_desc *pid = this_cpu_ptr(&posted_msi_pi_desc);
173
174	if (WARN_ON_ONCE(vector > NR_VECTORS || vector < FIRST_EXTERNAL_VECTOR))
175		return false;
176
177	return test_bit(vector, pid->pir);
178}
179
180extern void intel_posted_msi_init(void);
181#else
182static inline bool pi_pending_this_cpu(unsigned int vector) { return false; }
183
184static inline void intel_posted_msi_init(void) {};
185#endif /* X86_POSTED_MSI */
186
187#endif /* _X86_POSTED_INTR_H */