u64_stats_sync.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef _LINUX_U64_STATS_SYNC_H
  3#define _LINUX_U64_STATS_SYNC_H
  4
  5/*
  6 * Protect against 64-bit values tearing on 32-bit architectures. This is
  7 * typically used for statistics read/update in different subsystems.
  8 *
  9 * Key points :
 10 *
 11 * -  Use a seqcount on 32-bit
 12 * -  The whole thing is a no-op on 64-bit architectures.
 13 *
 14 * Usage constraints:
 15 *
 16 * 1) Write side must ensure mutual exclusion, or one seqcount update could
 17 *    be lost, thus blocking readers forever.
 18 *
 19 * 2) Write side must disable preemption, or a seqcount reader can preempt the
 20 *    writer and also spin forever.
 21 *
 22 * 3) Write side must use the _irqsave() variant if other writers, or a reader,
 23 *    can be invoked from an IRQ context. On 64bit systems this variant does not
 24 *    disable interrupts.
 25 *
 26 * 4) If reader fetches several counters, there is no guarantee the whole values
 27 *    are consistent w.r.t. each other (remember point #2: seqcounts are not
 28 *    used for 64bit architectures).
 29 *
 30 * 5) Readers are allowed to sleep or be preempted/interrupted: they perform
 31 *    pure reads.
 32 *
 33 * Usage :
 34 *
 35 * Stats producer (writer) should use following template granted it already got
 36 * an exclusive access to counters (a lock is already taken, or per cpu
 37 * data is used [in a non preemptable context])
 38 *
 39 *   spin_lock_bh(...) or other synchronization to get exclusive access
 40 *   ...
 41 *   u64_stats_update_begin(&stats->syncp);
 42 *   u64_stats_add(&stats->bytes64, len); // non atomic operation
 43 *   u64_stats_inc(&stats->packets64);    // non atomic operation
 44 *   u64_stats_update_end(&stats->syncp);
 45 *
 46 * While a consumer (reader) should use following template to get consistent
 47 * snapshot for each variable (but no guarantee on several ones)
 48 *
 49 * u64 tbytes, tpackets;
 50 * unsigned int start;
 51 *
 52 * do {
 53 *         start = u64_stats_fetch_begin(&stats->syncp);
 54 *         tbytes = u64_stats_read(&stats->bytes64); // non atomic operation
 55 *         tpackets = u64_stats_read(&stats->packets64); // non atomic operation
 56 * } while (u64_stats_fetch_retry(&stats->syncp, start));
 57 *
 58 *
 59 * Example of use in drivers/net/loopback.c, using per_cpu containers,
 60 * in BH disabled context.
 61 */
 62#include <linux/seqlock.h>
 63
 64struct u64_stats_sync {
 65#if BITS_PER_LONG == 32
 66	seqcount_t	seq;
 67#endif
 68};
 69
 70#if BITS_PER_LONG == 64
 71#include <asm/local64.h>
 72
 73typedef struct {
 74	local64_t	v;
 75} u64_stats_t ;
 76
 77static inline u64 u64_stats_read(const u64_stats_t *p)
 78{
 79	return local64_read(&p->v);
 80}
 81
 82static inline void u64_stats_set(u64_stats_t *p, u64 val)
 83{
 84	local64_set(&p->v, val);
 85}
 86
 87static inline void u64_stats_add(u64_stats_t *p, unsigned long val)
 88{
 89	local64_add(val, &p->v);
 90}
 91
 92static inline void u64_stats_sub(u64_stats_t *p, s64 val)
 93{
 94	local64_sub(val, &p->v);
 95}
 96
 97static inline void u64_stats_inc(u64_stats_t *p)
 98{
 99	local64_inc(&p->v);
100}
101
102static inline void u64_stats_init(struct u64_stats_sync *syncp) { }
103static inline void __u64_stats_update_begin(struct u64_stats_sync *syncp) { }
104static inline void __u64_stats_update_end(struct u64_stats_sync *syncp) { }
105static inline unsigned long __u64_stats_irqsave(void) { return 0; }
106static inline void __u64_stats_irqrestore(unsigned long flags) { }
107static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync *syncp)
108{
109	return 0;
110}
111static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
112					   unsigned int start)
113{
114	return false;
115}
116
117#else /* 64 bit */
118
119typedef struct {
120	u64		v;
121} u64_stats_t;
122
123static inline u64 u64_stats_read(const u64_stats_t *p)
124{
125	return p->v;
126}
127
128static inline void u64_stats_set(u64_stats_t *p, u64 val)
129{
130	p->v = val;
131}
132
133static inline void u64_stats_add(u64_stats_t *p, unsigned long val)
134{
135	p->v += val;
136}
137
138static inline void u64_stats_sub(u64_stats_t *p, s64 val)
139{
140	p->v -= val;
141}
142
143static inline void u64_stats_inc(u64_stats_t *p)
144{
145	p->v++;
146}
147
148#define u64_stats_init(syncp)				\
149	do {						\
150		struct u64_stats_sync *__s = (syncp);	\
151		seqcount_init(&__s->seq);		\
152	} while (0)
153
154static inline void __u64_stats_update_begin(struct u64_stats_sync *syncp)
155{
156	preempt_disable_nested();
157	write_seqcount_begin(&syncp->seq);
158}
159
160static inline void __u64_stats_update_end(struct u64_stats_sync *syncp)
161{
162	write_seqcount_end(&syncp->seq);
163	preempt_enable_nested();
164}
165
166static inline unsigned long __u64_stats_irqsave(void)
167{
168	unsigned long flags;
169
170	local_irq_save(flags);
171	return flags;
172}
173
174static inline void __u64_stats_irqrestore(unsigned long flags)
175{
176	local_irq_restore(flags);
177}
178
179static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync *syncp)
180{
181	return read_seqcount_begin(&syncp->seq);
182}
183
184static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
185					   unsigned int start)
186{
187	return read_seqcount_retry(&syncp->seq, start);
188}
189#endif /* !64 bit */
190
191static inline void u64_stats_update_begin(struct u64_stats_sync *syncp)
192{
193	__u64_stats_update_begin(syncp);
194}
195
196static inline void u64_stats_update_end(struct u64_stats_sync *syncp)
197{
198	__u64_stats_update_end(syncp);
199}
200
201static inline unsigned long u64_stats_update_begin_irqsave(struct u64_stats_sync *syncp)
202{
203	unsigned long flags = __u64_stats_irqsave();
204
205	__u64_stats_update_begin(syncp);
206	return flags;
207}
208
209static inline void u64_stats_update_end_irqrestore(struct u64_stats_sync *syncp,
210						   unsigned long flags)
211{
212	__u64_stats_update_end(syncp);
213	__u64_stats_irqrestore(flags);
214}
215
216static inline unsigned int u64_stats_fetch_begin(const struct u64_stats_sync *syncp)
217{
218	return __u64_stats_fetch_begin(syncp);
219}
220
221static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
222					 unsigned int start)
223{
224	return __u64_stats_fetch_retry(syncp, start);
225}
226
227#endif /* _LINUX_U64_STATS_SYNC_H */