swap.c 27.7 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7
/*
 *  linux/mm/swap.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 */

/*
Simon Arlott's avatar
Simon Arlott committed
8
 * This file contains the default values for the operation of the
Linus Torvalds's avatar
Linus Torvalds committed
9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 * Linux VM subsystem. Fine-tuning documentation can be found in
 * Documentation/sysctl/vm.txt.
 * Started 18.12.91
 * Swap aging added 23.2.95, Stephen Tweedie.
 * Buffermem limits added 12.3.98, Rik van Riel.
 */

#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/kernel_stat.h>
#include <linux/swap.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/pagevec.h>
#include <linux/init.h>
24
#include <linux/export.h>
Linus Torvalds's avatar
Linus Torvalds committed
25 26
#include <linux/mm_inline.h>
#include <linux/percpu_counter.h>
27
#include <linux/memremap.h>
Linus Torvalds's avatar
Linus Torvalds committed
28 29 30
#include <linux/percpu.h>
#include <linux/cpu.h>
#include <linux/notifier.h>
Peter Zijlstra's avatar
Peter Zijlstra committed
31
#include <linux/backing-dev.h>
32
#include <linux/memcontrol.h>
33
#include <linux/gfp.h>
34
#include <linux/uio.h>
35
#include <linux/hugetlb.h>
36
#include <linux/page_idle.h>
Linus Torvalds's avatar
Linus Torvalds committed
37

38 39
#include "internal.h"

40 41 42
#define CREATE_TRACE_POINTS
#include <trace/events/pagemap.h>

Linus Torvalds's avatar
Linus Torvalds committed
43 44 45
/* How many pages do we try to swap or page in/out together? */
int page_cluster;

46
static DEFINE_PER_CPU(struct pagevec, lru_add_pvec);
47
static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
48
static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs);
49
static DEFINE_PER_CPU(struct pagevec, lru_lazyfree_pvecs);
50 51 52
#ifdef CONFIG_SMP
static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
#endif
53

54 55 56 57
/*
 * This path almost never happens for VM activity - pages are normally
 * freed via pagevecs.  But it gets used by networking.
 */
58
static void __page_cache_release(struct page *page)
59 60 61
{
	if (PageLRU(page)) {
		struct zone *zone = page_zone(page);
62 63
		struct lruvec *lruvec;
		unsigned long flags;
64

65
		spin_lock_irqsave(zone_lru_lock(zone), flags);
66
		lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
67
		VM_BUG_ON_PAGE(!PageLRU(page), page);
68
		__ClearPageLRU(page);
69
		del_page_from_lru_list(page, lruvec, page_off_lru(page));
70
		spin_unlock_irqrestore(zone_lru_lock(zone), flags);
71
	}
72
	__ClearPageWaiters(page);
73
	mem_cgroup_uncharge(page);
74 75 76 77 78
}

static void __put_single_page(struct page *page)
{
	__page_cache_release(page);
79
	free_hot_cold_page(page, false);
80 81
}

82
static void __put_compound_page(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
83
{
84
	compound_page_dtor *dtor;
Linus Torvalds's avatar
Linus Torvalds committed
85

86 87 88 89 90 91 92 93
	/*
	 * __page_cache_release() is supposed to be called for thp, not for
	 * hugetlb. This is because hugetlb page does never have PageLRU set
	 * (it's never listed to any LRU lists) and no memcg routines should
	 * be called for hugetlb (it has a separate hugetlb_cgroup.)
	 */
	if (!PageHuge(page))
		__page_cache_release(page);
94 95 96 97
	dtor = get_compound_page_dtor(page);
	(*dtor)(page);
}

98
void __put_page(struct page *page)
99
{
100 101 102 103 104 105 106 107 108 109
	if (is_zone_device_page(page)) {
		put_dev_pagemap(page->pgmap);

		/*
		 * The page belongs to the device that created pgmap. Do
		 * not return it to page allocator.
		 */
		return;
	}

110
	if (unlikely(PageCompound(page)))
111 112
		__put_compound_page(page);
	else
113
		__put_single_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
114
}
115
EXPORT_SYMBOL(__put_page);
116

117
/**
118 119
 * put_pages_list() - release a list of pages
 * @pages: list of pages threaded on page->lru
120 121 122 123 124 125 126 127 128 129 130
 *
 * Release a list of pages which are strung together on page.lru.  Currently
 * used by read_cache_pages() and related error recovery code.
 */
void put_pages_list(struct list_head *pages)
{
	while (!list_empty(pages)) {
		struct page *victim;

		victim = list_entry(pages->prev, struct page, lru);
		list_del(&victim->lru);
131
		put_page(victim);
132 133 134 135
	}
}
EXPORT_SYMBOL(put_pages_list);

136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157
/*
 * get_kernel_pages() - pin kernel pages in memory
 * @kiov:	An array of struct kvec structures
 * @nr_segs:	number of segments to pin
 * @write:	pinning for read/write, currently ignored
 * @pages:	array that receives pointers to the pages pinned.
 *		Should be at least nr_segs long.
 *
 * Returns number of pages pinned. This may be fewer than the number
 * requested. If nr_pages is 0 or negative, returns 0. If no pages
 * were pinned, returns -errno. Each page returned must be released
 * with a put_page() call when it is finished with.
 */
int get_kernel_pages(const struct kvec *kiov, int nr_segs, int write,
		struct page **pages)
{
	int seg;

	for (seg = 0; seg < nr_segs; seg++) {
		if (WARN_ON(kiov[seg].iov_len != PAGE_SIZE))
			return seg;

158
		pages[seg] = kmap_to_page(kiov[seg].iov_base);
159
		get_page(pages[seg]);
160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
	}

	return seg;
}
EXPORT_SYMBOL_GPL(get_kernel_pages);

/*
 * get_kernel_page() - pin a kernel page in memory
 * @start:	starting kernel address
 * @write:	pinning for read/write, currently ignored
 * @pages:	array that receives pointer to the page pinned.
 *		Must be at least nr_segs long.
 *
 * Returns 1 if page is pinned. If the page was not pinned, returns
 * -errno. The page returned must be released with a put_page() call
 * when it is finished with.
 */
int get_kernel_page(unsigned long start, int write, struct page **pages)
{
	const struct kvec kiov = {
		.iov_base = (void *)start,
		.iov_len = PAGE_SIZE
	};

	return get_kernel_pages(&kiov, 1, write, pages);
}
EXPORT_SYMBOL_GPL(get_kernel_page);

Shaohua Li's avatar
Shaohua Li committed
188
static void pagevec_lru_move_fn(struct pagevec *pvec,
189 190
	void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg),
	void *arg)
191 192
{
	int i;
193
	struct pglist_data *pgdat = NULL;
194
	struct lruvec *lruvec;
Shaohua Li's avatar
Shaohua Li committed
195
	unsigned long flags = 0;
196 197 198

	for (i = 0; i < pagevec_count(pvec); i++) {
		struct page *page = pvec->pages[i];
199
		struct pglist_data *pagepgdat = page_pgdat(page);
200

201 202 203 204 205
		if (pagepgdat != pgdat) {
			if (pgdat)
				spin_unlock_irqrestore(&pgdat->lru_lock, flags);
			pgdat = pagepgdat;
			spin_lock_irqsave(&pgdat->lru_lock, flags);
206
		}
Shaohua Li's avatar
Shaohua Li committed
207

208
		lruvec = mem_cgroup_page_lruvec(page, pgdat);
209
		(*move_fn)(page, lruvec, arg);
210
	}
211 212
	if (pgdat)
		spin_unlock_irqrestore(&pgdat->lru_lock, flags);
213 214
	release_pages(pvec->pages, pvec->nr, pvec->cold);
	pagevec_reinit(pvec);
Shaohua Li's avatar
Shaohua Li committed
215 216
}

217 218
static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec,
				 void *arg)
Shaohua Li's avatar
Shaohua Li committed
219 220 221
{
	int *pgmoved = arg;

222 223 224 225
	if (PageLRU(page) && !PageUnevictable(page)) {
		del_page_from_lru_list(page, lruvec, page_lru(page));
		ClearPageActive(page);
		add_page_to_lru_list_tail(page, lruvec, page_lru(page));
Shaohua Li's avatar
Shaohua Li committed
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
		(*pgmoved)++;
	}
}

/*
 * pagevec_move_tail() must be called with IRQ disabled.
 * Otherwise this may cause nasty races.
 */
static void pagevec_move_tail(struct pagevec *pvec)
{
	int pgmoved = 0;

	pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved);
	__count_vm_events(PGROTATED, pgmoved);
}

Linus Torvalds's avatar
Linus Torvalds committed
242 243 244
/*
 * Writeback is about to end against a page which has been marked for immediate
 * reclaim.  If it still appears to be reclaimable, move it to the tail of the
245
 * inactive list.
Linus Torvalds's avatar
Linus Torvalds committed
246
 */
Shaohua Li's avatar
Shaohua Li committed
247
void rotate_reclaimable_page(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
248
{
249
	if (!PageLocked(page) && !PageDirty(page) &&
250
	    !PageUnevictable(page) && PageLRU(page)) {
251 252 253
		struct pagevec *pvec;
		unsigned long flags;

254
		get_page(page);
255
		local_irq_save(flags);
256
		pvec = this_cpu_ptr(&lru_rotate_pvecs);
257
		if (!pagevec_add(pvec, page) || PageCompound(page))
258 259 260
			pagevec_move_tail(pvec);
		local_irq_restore(flags);
	}
Linus Torvalds's avatar
Linus Torvalds committed
261 262
}

263
static void update_page_reclaim_stat(struct lruvec *lruvec,
264 265
				     int file, int rotated)
{
266
	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
267 268 269 270 271 272

	reclaim_stat->recent_scanned[file]++;
	if (rotated)
		reclaim_stat->recent_rotated[file]++;
}

273 274
static void __activate_page(struct page *page, struct lruvec *lruvec,
			    void *arg)
Linus Torvalds's avatar
Linus Torvalds committed
275
{
276
	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
277 278
		int file = page_is_file_cache(page);
		int lru = page_lru_base_type(page);
279

280
		del_page_from_lru_list(page, lruvec, lru);
281 282
		SetPageActive(page);
		lru += LRU_ACTIVE;
283
		add_page_to_lru_list(page, lruvec, lru);
284
		trace_mm_lru_activate(page);
285

286 287
		__count_vm_event(PGACTIVATE);
		update_page_reclaim_stat(lruvec, file, 1);
Linus Torvalds's avatar
Linus Torvalds committed
288
	}
289 290 291 292 293 294 295 296 297 298 299
}

#ifdef CONFIG_SMP
static void activate_page_drain(int cpu)
{
	struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu);

	if (pagevec_count(pvec))
		pagevec_lru_move_fn(pvec, __activate_page, NULL);
}

300 301 302 303 304
static bool need_activate_page_drain(int cpu)
{
	return pagevec_count(&per_cpu(activate_page_pvecs, cpu)) != 0;
}

305 306
void activate_page(struct page *page)
{
307
	page = compound_head(page);
308 309 310
	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
		struct pagevec *pvec = &get_cpu_var(activate_page_pvecs);

311
		get_page(page);
312
		if (!pagevec_add(pvec, page) || PageCompound(page))
313 314 315 316 317 318 319 320 321 322
			pagevec_lru_move_fn(pvec, __activate_page, NULL);
		put_cpu_var(activate_page_pvecs);
	}
}

#else
static inline void activate_page_drain(int cpu)
{
}

323 324 325 326 327
static bool need_activate_page_drain(int cpu)
{
	return false;
}

328 329 330 331
void activate_page(struct page *page)
{
	struct zone *zone = page_zone(page);

332
	page = compound_head(page);
333
	spin_lock_irq(zone_lru_lock(zone));
334
	__activate_page(page, mem_cgroup_page_lruvec(page, zone->zone_pgdat), NULL);
335
	spin_unlock_irq(zone_lru_lock(zone));
Linus Torvalds's avatar
Linus Torvalds committed
336
}
337
#endif
Linus Torvalds's avatar
Linus Torvalds committed
338

339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365
static void __lru_cache_activate_page(struct page *page)
{
	struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
	int i;

	/*
	 * Search backwards on the optimistic assumption that the page being
	 * activated has just been added to this pagevec. Note that only
	 * the local pagevec is examined as a !PageLRU page could be in the
	 * process of being released, reclaimed, migrated or on a remote
	 * pagevec that is currently being drained. Furthermore, marking
	 * a remote pagevec's page PageActive potentially hits a race where
	 * a page is marked PageActive just after it is added to the inactive
	 * list causing accounting errors and BUG_ON checks to trigger.
	 */
	for (i = pagevec_count(pvec) - 1; i >= 0; i--) {
		struct page *pagevec_page = pvec->pages[i];

		if (pagevec_page == page) {
			SetPageActive(page);
			break;
		}
	}

	put_cpu_var(lru_add_pvec);
}

Linus Torvalds's avatar
Linus Torvalds committed
366 367 368 369 370 371
/*
 * Mark a page as having seen activity.
 *
 * inactive,unreferenced	->	inactive,referenced
 * inactive,referenced		->	active,unreferenced
 * active,unreferenced		->	active,referenced
372 373 374
 *
 * When a newly allocated page is not yet visible, so safe for non-atomic ops,
 * __SetPageReferenced(page) may be substituted for mark_page_accessed(page).
Linus Torvalds's avatar
Linus Torvalds committed
375
 */
376
void mark_page_accessed(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
377
{
378
	page = compound_head(page);
379
	if (!PageActive(page) && !PageUnevictable(page) &&
380 381 382 383 384 385 386 387 388 389 390 391
			PageReferenced(page)) {

		/*
		 * If the page is on the LRU, queue it for activation via
		 * activate_page_pvecs. Otherwise, assume the page is on a
		 * pagevec, mark it active and it'll be moved to the active
		 * LRU on the next drain.
		 */
		if (PageLRU(page))
			activate_page(page);
		else
			__lru_cache_activate_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
392
		ClearPageReferenced(page);
393 394
		if (page_is_file_cache(page))
			workingset_activation(page);
Linus Torvalds's avatar
Linus Torvalds committed
395 396 397
	} else if (!PageReferenced(page)) {
		SetPageReferenced(page);
	}
398 399
	if (page_is_idle(page))
		clear_page_idle(page);
Linus Torvalds's avatar
Linus Torvalds committed
400 401 402
}
EXPORT_SYMBOL(mark_page_accessed);

403
static void __lru_cache_add(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
404
{
405 406
	struct pagevec *pvec = &get_cpu_var(lru_add_pvec);

407
	get_page(page);
408
	if (!pagevec_add(pvec, page) || PageCompound(page))
409
		__pagevec_lru_add(pvec);
410
	put_cpu_var(lru_add_pvec);
Linus Torvalds's avatar
Linus Torvalds committed
411
}
412 413 414 415 416 417 418

/**
 * lru_cache_add: add a page to the page lists
 * @page: the page to add
 */
void lru_cache_add_anon(struct page *page)
{
419 420
	if (PageActive(page))
		ClearPageActive(page);
421 422 423 424 425
	__lru_cache_add(page);
}

void lru_cache_add_file(struct page *page)
{
426 427
	if (PageActive(page))
		ClearPageActive(page);
428 429 430
	__lru_cache_add(page);
}
EXPORT_SYMBOL(lru_cache_add_file);
Linus Torvalds's avatar
Linus Torvalds committed
431

432
/**
433
 * lru_cache_add - add a page to a page list
434
 * @page: the page to be added to the LRU.
435 436 437 438 439
 *
 * Queue the page for addition to the LRU via pagevec. The decision on whether
 * to add the page to the [in]active [file|anon] list is deferred until the
 * pagevec is drained. This gives a chance for the caller of lru_cache_add()
 * have the page added to the active list using mark_page_accessed().
440
 */
441
void lru_cache_add(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
442
{
443 444
	VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page);
	VM_BUG_ON_PAGE(PageLRU(page), page);
445
	__lru_cache_add(page);
Linus Torvalds's avatar
Linus Torvalds committed
446 447
}

448 449 450 451 452 453 454 455 456 457 458 459
/**
 * add_page_to_unevictable_list - add a page to the unevictable list
 * @page:  the page to be added to the unevictable list
 *
 * Add page directly to its zone's unevictable list.  To avoid races with
 * tasks that might be making the page evictable, through eg. munlock,
 * munmap or exit, while it's not on the lru, we want to add the page
 * while it's locked or otherwise "invisible" to other tasks.  This is
 * difficult to do when using the pagevec cache, so bypass that.
 */
void add_page_to_unevictable_list(struct page *page)
{
460
	struct pglist_data *pgdat = page_pgdat(page);
461
	struct lruvec *lruvec;
462

463 464
	spin_lock_irq(&pgdat->lru_lock);
	lruvec = mem_cgroup_page_lruvec(page, pgdat);
465
	ClearPageActive(page);
466 467
	SetPageUnevictable(page);
	SetPageLRU(page);
468
	add_page_to_lru_list(page, lruvec, LRU_UNEVICTABLE);
469
	spin_unlock_irq(&pgdat->lru_lock);
470 471
}

472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505
/**
 * lru_cache_add_active_or_unevictable
 * @page:  the page to be added to LRU
 * @vma:   vma in which page is mapped for determining reclaimability
 *
 * Place @page on the active or unevictable LRU list, depending on its
 * evictability.  Note that if the page is not evictable, it goes
 * directly back onto it's zone's unevictable list, it does NOT use a
 * per cpu pagevec.
 */
void lru_cache_add_active_or_unevictable(struct page *page,
					 struct vm_area_struct *vma)
{
	VM_BUG_ON_PAGE(PageLRU(page), page);

	if (likely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED)) {
		SetPageActive(page);
		lru_cache_add(page);
		return;
	}

	if (!TestSetPageMlocked(page)) {
		/*
		 * We use the irq-unsafe __mod_zone_page_stat because this
		 * counter is not modified from interrupt context, and the pte
		 * lock is held(spinlock), which implies preemption disabled.
		 */
		__mod_zone_page_state(page_zone(page), NR_MLOCK,
				    hpage_nr_pages(page));
		count_vm_event(UNEVICTABLE_PGMLOCKED);
	}
	add_page_to_unevictable_list(page);
}

506 507 508 509 510 511
/*
 * If the page can not be invalidated, it is moved to the
 * inactive list to speed up its reclaim.  It is moved to the
 * head of the list, rather than the tail, to give the flusher
 * threads some time to write it out, as this is much more
 * effective than the single-page writeout from reclaim.
512 513 514 515 516 517 518 519 520 521 522 523 524 525
 *
 * If the page isn't page_mapped and dirty/writeback, the page
 * could reclaim asap using PG_reclaim.
 *
 * 1. active, mapped page -> none
 * 2. active, dirty/writeback page -> inactive, head, PG_reclaim
 * 3. inactive, mapped page -> none
 * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim
 * 5. inactive, clean -> inactive, tail
 * 6. Others -> none
 *
 * In 4, why it moves inactive's head, the VM expects the page would
 * be write it out by flusher threads as this is much more effective
 * than the single-page writeout from reclaim.
526
 */
527
static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec,
528
			      void *arg)
529 530
{
	int lru, file;
531
	bool active;
532

533
	if (!PageLRU(page))
534 535
		return;

536 537 538
	if (PageUnevictable(page))
		return;

539 540 541 542
	/* Some processes are using the page */
	if (page_mapped(page))
		return;

543
	active = PageActive(page);
544 545
	file = page_is_file_cache(page);
	lru = page_lru_base_type(page);
546 547

	del_page_from_lru_list(page, lruvec, lru + active);
548 549
	ClearPageActive(page);
	ClearPageReferenced(page);
550
	add_page_to_lru_list(page, lruvec, lru);
551

552 553 554 555 556 557 558 559 560 561 562 563
	if (PageWriteback(page) || PageDirty(page)) {
		/*
		 * PG_reclaim could be raced with end_page_writeback
		 * It can make readahead confusing.  But race window
		 * is _really_ small and  it's non-critical problem.
		 */
		SetPageReclaim(page);
	} else {
		/*
		 * The page's writeback ends up during pagevec
		 * We moves tha page into tail of inactive.
		 */
564
		list_move_tail(&page->lru, &lruvec->lists[lru]);
565 566 567 568 569
		__count_vm_event(PGROTATED);
	}

	if (active)
		__count_vm_event(PGDEACTIVATE);
570
	update_page_reclaim_stat(lruvec, file, 0);
571 572
}

573

574
static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec,
575 576
			    void *arg)
{
577
	if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&
578
	    !PageSwapCache(page) && !PageUnevictable(page)) {
579
		bool active = PageActive(page);
580

581 582
		del_page_from_lru_list(page, lruvec,
				       LRU_INACTIVE_ANON + active);
583 584
		ClearPageActive(page);
		ClearPageReferenced(page);
585 586 587 588 589 590 591
		/*
		 * lazyfree pages are clean anonymous pages. They have
		 * SwapBacked flag cleared to distinguish normal anonymous
		 * pages
		 */
		ClearPageSwapBacked(page);
		add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE);
592

593
		__count_vm_events(PGLAZYFREE, hpage_nr_pages(page));
594
		count_memcg_page_event(page, PGLAZYFREE);
595
		update_page_reclaim_stat(lruvec, 1, 0);
596 597 598
	}
}

599 600 601 602 603
/*
 * Drain pages out of the cpu's pagevecs.
 * Either "cpu" is the current CPU, and preemption has already been
 * disabled; or "cpu" is being hot-unplugged, and is already dead.
 */
604
void lru_add_drain_cpu(int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
605
{
606
	struct pagevec *pvec = &per_cpu(lru_add_pvec, cpu);
Linus Torvalds's avatar
Linus Torvalds committed
607

608
	if (pagevec_count(pvec))
609
		__pagevec_lru_add(pvec);
610 611 612 613 614 615 616 617 618 619

	pvec = &per_cpu(lru_rotate_pvecs, cpu);
	if (pagevec_count(pvec)) {
		unsigned long flags;

		/* No harm done if a racing interrupt already did this */
		local_irq_save(flags);
		pagevec_move_tail(pvec);
		local_irq_restore(flags);
	}
620

621
	pvec = &per_cpu(lru_deactivate_file_pvecs, cpu);
622
	if (pagevec_count(pvec))
623
		pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL);
624

625
	pvec = &per_cpu(lru_lazyfree_pvecs, cpu);
626
	if (pagevec_count(pvec))
627
		pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL);
628

629
	activate_page_drain(cpu);
630 631 632
}

/**
633
 * deactivate_file_page - forcefully deactivate a file page
634 635 636 637 638 639
 * @page: page to deactivate
 *
 * This function hints the VM that @page is a good reclaim candidate,
 * for example if its invalidation fails due to the page being dirty
 * or under writeback.
 */
640
void deactivate_file_page(struct page *page)
641
{
642
	/*
643 644
	 * In a workload with many unevictable page such as mprotect,
	 * unevictable page deactivation for accelerating reclaim is pointless.
645 646 647 648
	 */
	if (PageUnevictable(page))
		return;

649
	if (likely(get_page_unless_zero(page))) {
650
		struct pagevec *pvec = &get_cpu_var(lru_deactivate_file_pvecs);
651

652
		if (!pagevec_add(pvec, page) || PageCompound(page))
653 654
			pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL);
		put_cpu_var(lru_deactivate_file_pvecs);
655
	}
656 657
}

658
/**
659
 * mark_page_lazyfree - make an anon page lazyfree
660 661
 * @page: page to deactivate
 *
662 663
 * mark_page_lazyfree() moves @page to the inactive file list.
 * This is done to accelerate the reclaim of @page.
664
 */
665
void mark_page_lazyfree(struct page *page)
666
{
667
	if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&
668
	    !PageSwapCache(page) && !PageUnevictable(page)) {
669
		struct pagevec *pvec = &get_cpu_var(lru_lazyfree_pvecs);
670

671
		get_page(page);
672
		if (!pagevec_add(pvec, page) || PageCompound(page))
673 674
			pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL);
		put_cpu_var(lru_lazyfree_pvecs);
675 676 677
	}
}

678 679
void lru_add_drain(void)
{
680
	lru_add_drain_cpu(get_cpu());
681
	put_cpu();
Linus Torvalds's avatar
Linus Torvalds committed
682 683
}

684
static void lru_add_drain_per_cpu(struct work_struct *dummy)
685 686 687 688
{
	lru_add_drain();
}

689 690
static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);

691
void lru_add_drain_all_cpuslocked(void)
692
{
693 694 695 696
	static DEFINE_MUTEX(lock);
	static struct cpumask has_work;
	int cpu;

697 698 699 700 701 702 703
	/*
	 * Make sure nobody triggers this path before mm_percpu_wq is fully
	 * initialized.
	 */
	if (WARN_ON(!mm_percpu_wq))
		return;

704 705 706 707 708 709 710 711
	mutex_lock(&lock);
	cpumask_clear(&has_work);

	for_each_online_cpu(cpu) {
		struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);

		if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) ||
		    pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) ||
712
		    pagevec_count(&per_cpu(lru_deactivate_file_pvecs, cpu)) ||
713
		    pagevec_count(&per_cpu(lru_lazyfree_pvecs, cpu)) ||
714 715
		    need_activate_page_drain(cpu)) {
			INIT_WORK(work, lru_add_drain_per_cpu);
716
			queue_work_on(cpu, mm_percpu_wq, work);
717 718 719 720 721 722 723 724
			cpumask_set_cpu(cpu, &has_work);
		}
	}

	for_each_cpu(cpu, &has_work)
		flush_work(&per_cpu(lru_add_drain_work, cpu));

	mutex_unlock(&lock);
725 726
}

727 728 729 730 731 732 733
void lru_add_drain_all(void)
{
	get_online_cpus();
	lru_add_drain_all_cpuslocked();
	put_online_cpus();
}

734
/**
735
 * release_pages - batched put_page()
736 737 738
 * @pages: array of pages to release
 * @nr: number of pages
 * @cold: whether the pages are cache cold
Linus Torvalds's avatar
Linus Torvalds committed
739
 *
740 741
 * Decrement the reference count on all the pages in @pages.  If it
 * fell to zero, remove the page from the LRU and free it.
Linus Torvalds's avatar
Linus Torvalds committed
742
 */
743
void release_pages(struct page **pages, int nr, bool cold)
Linus Torvalds's avatar
Linus Torvalds committed
744 745
{
	int i;
746
	LIST_HEAD(pages_to_free);
747
	struct pglist_data *locked_pgdat = NULL;
748
	struct lruvec *lruvec;
749
	unsigned long uninitialized_var(flags);
750
	unsigned int uninitialized_var(lock_batch);
Linus Torvalds's avatar
Linus Torvalds committed
751 752 753 754

	for (i = 0; i < nr; i++) {
		struct page *page = pages[i];

755 756 757
		/*
		 * Make sure the IRQ-safe lock-holding time does not get
		 * excessive with a continuous string of pages from the
758
		 * same pgdat. The lock is held only if pgdat != NULL.
759
		 */
760 761 762
		if (locked_pgdat && ++lock_batch == SWAP_CLUSTER_MAX) {
			spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags);
			locked_pgdat = NULL;
763 764
		}

765
		if (is_huge_zero_page(page))
766 767
			continue;

768 769 770 771 772 773 774 775 776 777 778
		/* Device public page can not be huge page */
		if (is_device_public_page(page)) {
			if (locked_pgdat) {
				spin_unlock_irqrestore(&locked_pgdat->lru_lock,
						       flags);
				locked_pgdat = NULL;
			}
			put_zone_device_private_or_public_page(page);
			continue;
		}

779
		page = compound_head(page);
780
		if (!put_page_testzero(page))
Linus Torvalds's avatar
Linus Torvalds committed
781 782
			continue;

783
		if (PageCompound(page)) {
784 785 786
			if (locked_pgdat) {
				spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags);
				locked_pgdat = NULL;
787 788 789 790 791
			}
			__put_compound_page(page);
			continue;
		}

792
		if (PageLRU(page)) {
793
			struct pglist_data *pgdat = page_pgdat(page);
794

795 796 797
			if (pgdat != locked_pgdat) {
				if (locked_pgdat)
					spin_unlock_irqrestore(&locked_pgdat->lru_lock,
798
									flags);
799
				lock_batch = 0;
800 801
				locked_pgdat = pgdat;
				spin_lock_irqsave(&locked_pgdat->lru_lock, flags);
802
			}
803

804
			lruvec = mem_cgroup_page_lruvec(page, locked_pgdat);
805
			VM_BUG_ON_PAGE(!PageLRU(page), page);
806
			__ClearPageLRU(page);
807
			del_page_from_lru_list(page, lruvec, page_off_lru(page));
808 809
		}

810
		/* Clear Active bit in case of parallel mark_page_accessed */
811
		__ClearPageActive(page);
812
		__ClearPageWaiters(page);
813

814
		list_add(&page->lru, &pages_to_free);
Linus Torvalds's avatar
Linus Torvalds committed
815
	}
816 817
	if (locked_pgdat)
		spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
818

819
	mem_cgroup_uncharge_list(&pages_to_free);
820
	free_hot_cold_page_list(&pages_to_free, cold);
Linus Torvalds's avatar
Linus Torvalds committed
821
}
822
EXPORT_SYMBOL(release_pages);
Linus Torvalds's avatar
Linus Torvalds committed
823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839

/*
 * The pages which we're about to release may be in the deferred lru-addition
 * queues.  That would prevent them from really being freed right now.  That's
 * OK from a correctness point of view but is inefficient - those pages may be
 * cache-warm and we want to give them back to the page allocator ASAP.
 *
 * So __pagevec_release() will drain those queues here.  __pagevec_lru_add()
 * and __pagevec_lru_add_active() call release_pages() directly to avoid
 * mutual recursion.
 */
void __pagevec_release(struct pagevec *pvec)
{
	lru_add_drain();
	release_pages(pvec->pages, pagevec_count(pvec), pvec->cold);
	pagevec_reinit(pvec);
}
840 841
EXPORT_SYMBOL(__pagevec_release);

842
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
843
/* used by __split_huge_page_refcount() */
844
void lru_add_page_tail(struct page *page, struct page *page_tail,
845
		       struct lruvec *lruvec, struct list_head *list)
846 847 848
{
	const int file = 0;

849 850 851
	VM_BUG_ON_PAGE(!PageHead(page), page);
	VM_BUG_ON_PAGE(PageCompound(page_tail), page);
	VM_BUG_ON_PAGE(PageLRU(page_tail), page);
852
	VM_BUG_ON(NR_CPUS != 1 &&
853
		  !spin_is_locked(&lruvec_pgdat(lruvec)->lru_lock));
854

855 856
	if (!list)
		SetPageLRU(page_tail);
857

858 859
	if (likely(PageLRU(page)))
		list_add_tail(&page_tail->lru, &page->lru);
860 861 862 863 864
	else if (list) {
		/* page reclaim is reclaiming a huge page */
		get_page(page_tail);
		list_add_tail(&page_tail->lru, list);
	} else {
865 866 867 868 869 870 871 872
		struct list_head *list_head;
		/*
		 * Head page has not yet been counted, as an hpage,
		 * so we must account for each subpage individually.
		 *
		 * Use the standard add function to put page_tail on the list,
		 * but then correct its position so they all end up in order.
		 */
873
		add_page_to_lru_list(page_tail, lruvec, page_lru(page_tail));
874 875
		list_head = page_tail->lru.prev;
		list_move_tail(&page_tail->lru, list_head);
876
	}
877 878

	if (!PageUnevictable(page))
879
		update_page_reclaim_stat(lruvec, file, PageActive(page_tail));
880
}
881
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
882

883 884
static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec,
				 void *arg)
Shaohua Li's avatar
Shaohua Li committed
885
{
886 887 888
	int file = page_is_file_cache(page);
	int active = PageActive(page);
	enum lru_list lru = page_lru(page);
Shaohua Li's avatar
Shaohua Li committed
889

890
	VM_BUG_ON_PAGE(PageLRU(page), page);
Shaohua Li's avatar
Shaohua Li committed
891 892

	SetPageLRU(page);
893 894
	add_page_to_lru_list(page, lruvec, lru);
	update_page_reclaim_stat(lruvec, file, active);
895
	trace_mm_lru_insertion(page, lru);
Shaohua Li's avatar
Shaohua Li committed
896 897
}

Linus Torvalds's avatar
Linus Torvalds committed
898 899 900 901
/*
 * Add the passed pages to the LRU, then drop the caller's refcount
 * on them.  Reinitialises the caller's pagevec.
 */
902
void __pagevec_lru_add(struct pagevec *pvec)
Linus Torvalds's avatar
Linus Torvalds committed
903
{
904
	pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
905
}
906
EXPORT_SYMBOL(__pagevec_lru_add);
Linus Torvalds's avatar
Linus Torvalds committed
907

908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958
/**
 * pagevec_lookup_entries - gang pagecache lookup
 * @pvec:	Where the resulting entries are placed
 * @mapping:	The address_space to search
 * @start:	The starting entry index
 * @nr_entries:	The maximum number of entries
 * @indices:	The cache indices corresponding to the entries in @pvec
 *
 * pagevec_lookup_entries() will search for and return a group of up
 * to @nr_entries pages and shadow entries in the mapping.  All
 * entries are placed in @pvec.  pagevec_lookup_entries() takes a
 * reference against actual pages in @pvec.
 *
 * The search returns a group of mapping-contiguous entries with
 * ascending indexes.  There may be holes in the indices due to
 * not-present entries.
 *
 * pagevec_lookup_entries() returns the number of entries which were
 * found.
 */
unsigned pagevec_lookup_entries(struct pagevec *pvec,
				struct address_space *mapping,
				pgoff_t start, unsigned nr_pages,
				pgoff_t *indices)
{
	pvec->nr = find_get_entries(mapping, start, nr_pages,
				    pvec->pages, indices);
	return pagevec_count(pvec);
}

/**
 * pagevec_remove_exceptionals - pagevec exceptionals pruning
 * @pvec:	The pagevec to prune
 *
 * pagevec_lookup_entries() fills both pages and exceptional radix
 * tree entries into the pagevec.  This function prunes all
 * exceptionals from @pvec without leaving holes, so that it can be
 * passed on to page-only pagevec operations.
 */
void pagevec_remove_exceptionals(struct pagevec *pvec)
{
	int i, j;

	for (i = 0, j = 0; i < pagevec_count(pvec); i++) {
		struct page *page = pvec->pages[i];
		if (!radix_tree_exceptional_entry(page))
			pvec->pages[j++] = page;
	}
	pvec->nr = j;
}

Linus Torvalds's avatar
Linus Torvalds committed
959
/**
960
 * pagevec_lookup_range - gang pagecache lookup
Linus Torvalds's avatar
Linus Torvalds committed
961 962 963
 * @pvec:	Where the resulting pages are placed
 * @mapping:	The address_space to search
 * @start:	The starting page index
964
 * @end:	The final page index
Linus Torvalds's avatar
Linus Torvalds committed
965 966
 * @nr_pages:	The maximum number of pages
 *
967 968 969
 * pagevec_lookup_range() will search for and return a group of up to @nr_pages
 * pages in the mapping starting from index @start and upto index @end
 * (inclusive).  The pages are placed in @pvec.  pagevec_lookup() takes a
Linus Torvalds's avatar
Linus Torvalds committed
970 971 972
 * reference against the pages in @pvec.
 *
 * The search returns a group of mapping-contiguous pages with ascending
973 974
 * indexes.  There may be holes in the indices due to not-present pages. We
 * also update @start to index the next page for the traversal.
Linus Torvalds's avatar
Linus Torvalds committed
975
 *
976 977 978
 * pagevec_lookup_range() returns the number of pages which were found. If this
 * number is smaller than @nr_pages, the end of specified range has been
 * reached.
Linus Torvalds's avatar
Linus Torvalds committed
979
 */
980
unsigned pagevec_lookup_range(struct pagevec *pvec,
981
		struct address_space *mapping, pgoff_t *start, pgoff_t end)
Linus Torvalds's avatar
Linus Torvalds committed
982
{
983
	pvec->nr = find_get_pages_range(mapping, start, end, PAGEVEC_SIZE,
984
					pvec->pages);
Linus Torvalds's avatar
Linus Torvalds committed
985 986
	return pagevec_count(pvec);
}
987
EXPORT_SYMBOL(pagevec_lookup_range);
988

Linus Torvalds's avatar
Linus Torvalds committed
989 990 991 992 993 994 995
unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping,
		pgoff_t *index, int tag, unsigned nr_pages)
{
	pvec->nr = find_get_pages_tag(mapping, index, tag,
					nr_pages, pvec->pages);
	return pagevec_count(pvec);
}
996
EXPORT_SYMBOL(pagevec_lookup_tag);
Linus Torvalds's avatar
Linus Torvalds committed
997 998 999 1000 1001 1002

/*
 * Perform any setup for the swap system
 */
void __init swap_setup(void)
{
1003
	unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT);
Peter Zijlstra's avatar
Peter Zijlstra committed
1004

Linus Torvalds's avatar
Linus Torvalds committed
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
	/* Use a smaller cluster for small-memory machines */
	if (megs < 16)
		page_cluster = 2;
	else
		page_cluster = 3;
	/*
	 * Right now other parts of the system means that we
	 * _really_ don't want to cluster much more
	 */
}