swap_cgroup.c 5.21 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2
#include <linux/swap_cgroup.h>
3
#include <linux/vmalloc.h>
4
#include <linux/mm.h>
5

6
#include <linux/swapops.h> /* depends on mm.h include */
7 8 9 10 11

static DEFINE_MUTEX(swap_cgroup_mutex);
struct swap_cgroup_ctrl {
	struct page **map;
	unsigned long length;
12
	spinlock_t	lock;
13 14
};

15
static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
16 17

struct swap_cgroup {
18
	unsigned short		id;
19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
};
#define SC_PER_PAGE	(PAGE_SIZE/sizeof(struct swap_cgroup))

/*
 * SwapCgroup implements "lookup" and "exchange" operations.
 * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
 * against SwapCache. At swap_free(), this is accessed directly from swap.
 *
 * This means,
 *  - we have no race in "exchange" when we're accessed via SwapCache because
 *    SwapCache(and its swp_entry) is under lock.
 *  - When called via swap_free(), there is no user of this entry and no race.
 * Then, we don't need lock around "exchange".
 *
 * TODO: we can push these buffers out to HIGHMEM.
 */

/*
 * allocate buffer for swap_cgroup.
 */
static int swap_cgroup_prepare(int type)
{
	struct page *page;
	struct swap_cgroup_ctrl *ctrl;
	unsigned long idx, max;

	ctrl = &swap_cgroup_ctrl[type];

	for (idx = 0; idx < ctrl->length; idx++) {
		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
		if (!page)
			goto not_enough_page;
		ctrl->map[idx] = page;
52 53 54

		if (!(idx % SWAP_CLUSTER_MAX))
			cond_resched();
55 56 57 58 59 60 61 62 63 64
	}
	return 0;
not_enough_page:
	max = idx;
	for (idx = 0; idx < max; idx++)
		__free_page(ctrl->map[idx]);

	return -ENOMEM;
}

65 66 67 68 69 70 71 72 73 74 75
static struct swap_cgroup *__lookup_swap_cgroup(struct swap_cgroup_ctrl *ctrl,
						pgoff_t offset)
{
	struct page *mappage;
	struct swap_cgroup *sc;

	mappage = ctrl->map[offset / SC_PER_PAGE];
	sc = page_address(mappage);
	return sc + offset % SC_PER_PAGE;
}

76 77 78 79 80 81 82 83 84
static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
					struct swap_cgroup_ctrl **ctrlp)
{
	pgoff_t offset = swp_offset(ent);
	struct swap_cgroup_ctrl *ctrl;

	ctrl = &swap_cgroup_ctrl[swp_type(ent)];
	if (ctrlp)
		*ctrlp = ctrl;
85
	return __lookup_swap_cgroup(ctrl, offset);
86 87
}

88 89
/**
 * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
Wanpeng Li's avatar
Wanpeng Li committed
90
 * @ent: swap entry to be cmpxchged
91 92 93 94
 * @old: old id
 * @new: new id
 *
 * Returns old id at success, 0 at failure.
Lucas De Marchi's avatar
Lucas De Marchi committed
95
 * (There is no mem_cgroup using 0 as its id)
96 97 98 99 100 101
 */
unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
					unsigned short old, unsigned short new)
{
	struct swap_cgroup_ctrl *ctrl;
	struct swap_cgroup *sc;
102 103
	unsigned long flags;
	unsigned short retval;
104

105
	sc = lookup_swap_cgroup(ent, &ctrl);
106

107 108 109 110
	spin_lock_irqsave(&ctrl->lock, flags);
	retval = sc->id;
	if (retval == old)
		sc->id = new;
111
	else
112 113 114
		retval = 0;
	spin_unlock_irqrestore(&ctrl->lock, flags);
	return retval;
115 116
}

117
/**
118 119
 * swap_cgroup_record - record mem_cgroup for a set of swap entries
 * @ent: the first swap entry to be recorded into
Wanpeng Li's avatar
Wanpeng Li committed
120
 * @id: mem_cgroup to be recorded
121
 * @nr_ents: number of swap entries to be recorded
122
 *
123 124
 * Returns old value at success, 0 at failure.
 * (Of course, old value can be 0.)
125
 */
126 127
unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id,
				  unsigned int nr_ents)
128 129 130
{
	struct swap_cgroup_ctrl *ctrl;
	struct swap_cgroup *sc;
131
	unsigned short old;
132
	unsigned long flags;
133 134
	pgoff_t offset = swp_offset(ent);
	pgoff_t end = offset + nr_ents;
135

136
	sc = lookup_swap_cgroup(ent, &ctrl);
137

138 139
	spin_lock_irqsave(&ctrl->lock, flags);
	old = sc->id;
140 141 142 143 144 145 146 147 148 149 150
	for (;;) {
		VM_BUG_ON(sc->id != old);
		sc->id = id;
		offset++;
		if (offset == end)
			break;
		if (offset % SC_PER_PAGE)
			sc++;
		else
			sc = __lookup_swap_cgroup(ctrl, offset);
	}
151
	spin_unlock_irqrestore(&ctrl->lock, flags);
152 153 154 155 156

	return old;
}

/**
157
 * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
158 159
 * @ent: swap entry to be looked up.
 *
160
 * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
161
 */
162
unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
163
{
164
	return lookup_swap_cgroup(ent, NULL)->id;
165 166 167 168 169 170 171 172 173 174 175 176
}

int swap_cgroup_swapon(int type, unsigned long max_pages)
{
	void *array;
	unsigned long array_size;
	unsigned long length;
	struct swap_cgroup_ctrl *ctrl;

	if (!do_swap_account)
		return 0;

177
	length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
178 179
	array_size = length * sizeof(void *);

180
	array = vzalloc(array_size);
181 182 183 184 185 186 187
	if (!array)
		goto nomem;

	ctrl = &swap_cgroup_ctrl[type];
	mutex_lock(&swap_cgroup_mutex);
	ctrl->length = length;
	ctrl->map = array;
188
	spin_lock_init(&ctrl->lock);
189 190 191 192 193
	if (swap_cgroup_prepare(type)) {
		/* memory shortage */
		ctrl->map = NULL;
		ctrl->length = 0;
		mutex_unlock(&swap_cgroup_mutex);
194
		vfree(array);
195 196 197 198 199 200
		goto nomem;
	}
	mutex_unlock(&swap_cgroup_mutex);

	return 0;
nomem:
201 202
	pr_info("couldn't allocate enough memory for swap_cgroup\n");
	pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
203 204 205 206 207
	return -ENOMEM;
}

void swap_cgroup_swapoff(int type)
{
208 209
	struct page **map;
	unsigned long i, length;
210 211 212 213 214 215 216
	struct swap_cgroup_ctrl *ctrl;

	if (!do_swap_account)
		return;

	mutex_lock(&swap_cgroup_mutex);
	ctrl = &swap_cgroup_ctrl[type];
217 218 219 220 221 222 223 224 225
	map = ctrl->map;
	length = ctrl->length;
	ctrl->map = NULL;
	ctrl->length = 0;
	mutex_unlock(&swap_cgroup_mutex);

	if (map) {
		for (i = 0; i < length; i++) {
			struct page *page = map[i];
226 227
			if (page)
				__free_page(page);
228 229
			if (!(i % SWAP_CLUSTER_MAX))
				cond_resched();
230
		}
231
		vfree(map);
232 233
	}
}