blk-cgroup-rwstat.h

  1/* SPDX-License-Identifier: GPL-2.0
  2 *
  3 * Legacy blkg rwstat helpers enabled by CONFIG_BLK_CGROUP_RWSTAT.
  4 * Do not use in new code.
  5 */
  6#ifndef _BLK_CGROUP_RWSTAT_H
  7#define _BLK_CGROUP_RWSTAT_H
  8
  9#include "blk-cgroup.h"
 10
 11enum blkg_rwstat_type {
 12	BLKG_RWSTAT_READ,
 13	BLKG_RWSTAT_WRITE,
 14	BLKG_RWSTAT_SYNC,
 15	BLKG_RWSTAT_ASYNC,
 16	BLKG_RWSTAT_DISCARD,
 17
 18	BLKG_RWSTAT_NR,
 19	BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
 20};
 21
 22/*
 23 * blkg_[rw]stat->aux_cnt is excluded for local stats but included for
 24 * recursive.  Used to carry stats of dead children.
 25 */
 26struct blkg_rwstat {
 27	struct percpu_counter		cpu_cnt[BLKG_RWSTAT_NR];
 28	atomic64_t			aux_cnt[BLKG_RWSTAT_NR];
 29};
 30
 31struct blkg_rwstat_sample {
 32	u64				cnt[BLKG_RWSTAT_NR];
 33};
 34
 35static inline u64 blkg_rwstat_read_counter(struct blkg_rwstat *rwstat,
 36		unsigned int idx)
 37{
 38	return atomic64_read(&rwstat->aux_cnt[idx]) +
 39		percpu_counter_sum_positive(&rwstat->cpu_cnt[idx]);
 40}
 41
 42int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp);
 43void blkg_rwstat_exit(struct blkg_rwstat *rwstat);
 44u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
 45			 const struct blkg_rwstat_sample *rwstat);
 46u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
 47		       int off);
 48void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol,
 49		int off, struct blkg_rwstat_sample *sum);
 50
 51
 52/**
 53 * blkg_rwstat_add - add a value to a blkg_rwstat
 54 * @rwstat: target blkg_rwstat
 55 * @opf: REQ_OP and flags
 56 * @val: value to add
 57 *
 58 * Add @val to @rwstat.  The counters are chosen according to @rw.  The
 59 * caller is responsible for synchronizing calls to this function.
 60 */
 61static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
 62				   blk_opf_t opf, uint64_t val)
 63{
 64	struct percpu_counter *cnt;
 65
 66	if (op_is_discard(opf))
 67		cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_DISCARD];
 68	else if (op_is_write(opf))
 69		cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE];
 70	else
 71		cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ];
 72
 73	percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
 74
 75	if (op_is_sync(opf))
 76		cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC];
 77	else
 78		cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC];
 79
 80	percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
 81}
 82
 83/**
 84 * blkg_rwstat_read - read the current values of a blkg_rwstat
 85 * @rwstat: blkg_rwstat to read
 86 * @result: where to put the current values
 87 *
 88 * Read the current snapshot of @rwstat and return it in the @result counts.
 89 */
 90static inline void blkg_rwstat_read(struct blkg_rwstat *rwstat,
 91		struct blkg_rwstat_sample *result)
 92{
 93	int i;
 94
 95	for (i = 0; i < BLKG_RWSTAT_NR; i++)
 96		result->cnt[i] =
 97			percpu_counter_sum_positive(&rwstat->cpu_cnt[i]);
 98}
 99
100/**
101 * blkg_rwstat_total - read the total count of a blkg_rwstat
102 * @rwstat: blkg_rwstat to read
103 *
104 * Return the total count of @rwstat regardless of the IO direction.  This
105 * function can be called without synchronization and takes care of u64
106 * atomicity.
107 */
108static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)
109{
110	struct blkg_rwstat_sample tmp = { };
111
112	blkg_rwstat_read(rwstat, &tmp);
113	return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
114}
115
116/**
117 * blkg_rwstat_reset - reset a blkg_rwstat
118 * @rwstat: blkg_rwstat to reset
119 */
120static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
121{
122	int i;
123
124	for (i = 0; i < BLKG_RWSTAT_NR; i++) {
125		percpu_counter_set(&rwstat->cpu_cnt[i], 0);
126		atomic64_set(&rwstat->aux_cnt[i], 0);
127	}
128}
129
130/**
131 * blkg_rwstat_add_aux - add a blkg_rwstat into another's aux count
132 * @to: the destination blkg_rwstat
133 * @from: the source
134 *
135 * Add @from's count including the aux one to @to's aux count.
136 */
137static inline void blkg_rwstat_add_aux(struct blkg_rwstat *to,
138				       struct blkg_rwstat *from)
139{
140	u64 sum[BLKG_RWSTAT_NR];
141	int i;
142
143	for (i = 0; i < BLKG_RWSTAT_NR; i++)
144		sum[i] = percpu_counter_sum_positive(&from->cpu_cnt[i]);
145
146	for (i = 0; i < BLKG_RWSTAT_NR; i++)
147		atomic64_add(sum[i] + atomic64_read(&from->aux_cnt[i]),
148			     &to->aux_cnt[i]);
149}
150#endif	/* _BLK_CGROUP_RWSTAT_H */