swap_cgroup.c

  1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/swap_cgroup.h>
  3#include <linux/vmalloc.h>
  4#include <linux/mm.h>
  5
  6#include <linux/swapops.h> /* depends on mm.h include */
  7
  8static DEFINE_MUTEX(swap_cgroup_mutex);
  9
 10/* Pack two cgroup id (short) of two entries in one swap_cgroup (atomic_t) */
 11#define ID_PER_SC (sizeof(struct swap_cgroup) / sizeof(unsigned short))
 12#define ID_SHIFT (BITS_PER_TYPE(unsigned short))
 13#define ID_MASK (BIT(ID_SHIFT) - 1)
 14struct swap_cgroup {
 15	atomic_t ids;
 16};
 17
 18struct swap_cgroup_ctrl {
 19	struct swap_cgroup *map;
 20};
 21
 22static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
 23
 24static unsigned short __swap_cgroup_id_lookup(struct swap_cgroup *map,
 25					      pgoff_t offset)
 26{
 27	unsigned int shift = (offset % ID_PER_SC) * ID_SHIFT;
 28	unsigned int old_ids = atomic_read(&map[offset / ID_PER_SC].ids);
 29
 30	BUILD_BUG_ON(!is_power_of_2(ID_PER_SC));
 31	BUILD_BUG_ON(sizeof(struct swap_cgroup) != sizeof(atomic_t));
 32
 33	return (old_ids >> shift) & ID_MASK;
 34}
 35
 36static unsigned short __swap_cgroup_id_xchg(struct swap_cgroup *map,
 37					    pgoff_t offset,
 38					    unsigned short new_id)
 39{
 40	unsigned short old_id;
 41	struct swap_cgroup *sc = &map[offset / ID_PER_SC];
 42	unsigned int shift = (offset % ID_PER_SC) * ID_SHIFT;
 43	unsigned int new_ids, old_ids = atomic_read(&sc->ids);
 44
 45	do {
 46		old_id = (old_ids >> shift) & ID_MASK;
 47		new_ids = (old_ids & ~(ID_MASK << shift));
 48		new_ids |= ((unsigned int)new_id) << shift;
 49	} while (!atomic_try_cmpxchg(&sc->ids, &old_ids, new_ids));
 50
 51	return old_id;
 52}
 53
 54/**
 55 * swap_cgroup_record - record mem_cgroup for a set of swap entries.
 56 * These entries must belong to one single folio, and that folio
 57 * must be being charged for swap space (swap out), and these
 58 * entries must not have been charged
 59 *
 60 * @folio: the folio that the swap entry belongs to
 61 * @id: mem_cgroup ID to be recorded
 62 * @ent: the first swap entry to be recorded
 63 */
 64void swap_cgroup_record(struct folio *folio, unsigned short id,
 65			swp_entry_t ent)
 66{
 67	unsigned int nr_ents = folio_nr_pages(folio);
 68	struct swap_cgroup *map;
 69	pgoff_t offset, end;
 70	unsigned short old;
 71
 72	offset = swp_offset(ent);
 73	end = offset + nr_ents;
 74	map = swap_cgroup_ctrl[swp_type(ent)].map;
 75
 76	do {
 77		old = __swap_cgroup_id_xchg(map, offset, id);
 78		VM_BUG_ON(old);
 79	} while (++offset != end);
 80}
 81
 82/**
 83 * swap_cgroup_clear - clear mem_cgroup for a set of swap entries.
 84 * These entries must be being uncharged from swap. They either
 85 * belongs to one single folio in the swap cache (swap in for
 86 * cgroup v1), or no longer have any users (slot freeing).
 87 *
 88 * @ent: the first swap entry to be recorded into
 89 * @nr_ents: number of swap entries to be recorded
 90 *
 91 * Returns the existing old value.
 92 */
 93unsigned short swap_cgroup_clear(swp_entry_t ent, unsigned int nr_ents)
 94{
 95	pgoff_t offset, end;
 96	struct swap_cgroup *map;
 97	unsigned short old, iter = 0;
 98
 99	offset = swp_offset(ent);
100	end = offset + nr_ents;
101	map = swap_cgroup_ctrl[swp_type(ent)].map;
102
103	do {
104		old = __swap_cgroup_id_xchg(map, offset, 0);
105		if (!iter)
106			iter = old;
107		VM_BUG_ON(iter != old);
108	} while (++offset != end);
109
110	return old;
111}
112
113/**
114 * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
115 * @ent: swap entry to be looked up.
116 *
117 * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
118 */
119unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
120{
121	struct swap_cgroup_ctrl *ctrl;
122
123	if (mem_cgroup_disabled())
124		return 0;
125
126	ctrl = &swap_cgroup_ctrl[swp_type(ent)];
127	return __swap_cgroup_id_lookup(ctrl->map, swp_offset(ent));
128}
129
130int swap_cgroup_swapon(int type, unsigned long max_pages)
131{
132	struct swap_cgroup *map;
133	struct swap_cgroup_ctrl *ctrl;
134
135	if (mem_cgroup_disabled())
136		return 0;
137
138	BUILD_BUG_ON(sizeof(unsigned short) * ID_PER_SC !=
139		     sizeof(struct swap_cgroup));
140	map = vzalloc(DIV_ROUND_UP(max_pages, ID_PER_SC) *
141		      sizeof(struct swap_cgroup));
142	if (!map)
143		goto nomem;
144
145	ctrl = &swap_cgroup_ctrl[type];
146	mutex_lock(&swap_cgroup_mutex);
147	ctrl->map = map;
148	mutex_unlock(&swap_cgroup_mutex);
149
150	return 0;
151nomem:
152	pr_info("couldn't allocate enough memory for swap_cgroup\n");
153	pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
154	return -ENOMEM;
155}
156
157void swap_cgroup_swapoff(int type)
158{
159	struct swap_cgroup *map;
160	struct swap_cgroup_ctrl *ctrl;
161
162	if (mem_cgroup_disabled())
163		return;
164
165	mutex_lock(&swap_cgroup_mutex);
166	ctrl = &swap_cgroup_ctrl[type];
167	map = ctrl->map;
168	ctrl->map = NULL;
169	mutex_unlock(&swap_cgroup_mutex);
170
171	vfree(map);
172}