vmscan.c 116 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
Linus Torvalds's avatar
Linus Torvalds committed
2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 *  linux/mm/vmscan.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *
 *  Swap reorganised 29.12.95, Stephen Tweedie.
 *  kswapd added: 7.1.96  sct
 *  Removed kswapd_ctl limits, and swap out as many pages as needed
 *  to bring the system back to freepages.high: 2.4.97, Rik van Riel.
 *  Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
 *  Multiqueue VM started 5.8.00, Rik van Riel.
 */

15 16
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

Linus Torvalds's avatar
Linus Torvalds committed
17
#include <linux/mm.h>
18
#include <linux/sched/mm.h>
Linus Torvalds's avatar
Linus Torvalds committed
19
#include <linux/module.h>
20
#include <linux/gfp.h>
Linus Torvalds's avatar
Linus Torvalds committed
21 22 23 24 25
#include <linux/kernel_stat.h>
#include <linux/swap.h>
#include <linux/pagemap.h>
#include <linux/init.h>
#include <linux/highmem.h>
26
#include <linux/vmpressure.h>
27
#include <linux/vmstat.h>
Linus Torvalds's avatar
Linus Torvalds committed
28 29 30 31 32 33 34 35 36 37 38
#include <linux/file.h>
#include <linux/writeback.h>
#include <linux/blkdev.h>
#include <linux/buffer_head.h>	/* for try_to_release_page(),
					buffer_heads_over_limit */
#include <linux/mm_inline.h>
#include <linux/backing-dev.h>
#include <linux/rmap.h>
#include <linux/topology.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
39
#include <linux/compaction.h>
Linus Torvalds's avatar
Linus Torvalds committed
40 41
#include <linux/notifier.h>
#include <linux/rwsem.h>
42
#include <linux/delay.h>
43
#include <linux/kthread.h>
44
#include <linux/freezer.h>
45
#include <linux/memcontrol.h>
46
#include <linux/delayacct.h>
47
#include <linux/sysctl.h>
48
#include <linux/oom.h>
49
#include <linux/prefetch.h>
50
#include <linux/printk.h>
51
#include <linux/dax.h>
Linus Torvalds's avatar
Linus Torvalds committed
52 53 54 55 56

#include <asm/tlbflush.h>
#include <asm/div64.h>

#include <linux/swapops.h>
57
#include <linux/balloon_compaction.h>
Linus Torvalds's avatar
Linus Torvalds committed
58

59 60
#include "internal.h"

61 62 63
#define CREATE_TRACE_POINTS
#include <trace/events/vmscan.h>

Linus Torvalds's avatar
Linus Torvalds committed
64
struct scan_control {
65 66 67
	/* How many pages shrink_list() should reclaim */
	unsigned long nr_to_reclaim;

Linus Torvalds's avatar
Linus Torvalds committed
68
	/* This context's GFP mask */
Al Viro's avatar
Al Viro committed
69
	gfp_t gfp_mask;
Linus Torvalds's avatar
Linus Torvalds committed
70

71
	/* Allocation order */
Andy Whitcroft's avatar
Andy Whitcroft committed
72
	int order;
73

74 75 76 77 78
	/*
	 * Nodemask of nodes allowed by the caller. If NULL, all nodes
	 * are scanned.
	 */
	nodemask_t	*nodemask;
79

80 81 82 83 84
	/*
	 * The memory cgroup that hit its limit and as a result is the
	 * primary target of this reclaim invocation.
	 */
	struct mem_cgroup *target_mem_cgroup;
85

86 87 88
	/* Scan (total_size >> priority) pages at once */
	int priority;

89 90 91
	/* The highest zone to isolate pages for reclaim from */
	enum zone_type reclaim_idx;

92
	/* Writepage batching in laptop mode; RECLAIM_WRITE */
93 94 95 96 97 98 99 100
	unsigned int may_writepage:1;

	/* Can mapped pages be reclaimed? */
	unsigned int may_unmap:1;

	/* Can pages be swapped as part of reclaim? */
	unsigned int may_swap:1;

101 102 103 104 105 106 107
	/*
	 * Cgroups are not reclaimed below their configured memory.low,
	 * unless we threaten to OOM. If any cgroups are skipped due to
	 * memory.low and nothing was reclaimed, go back for memory.low.
	 */
	unsigned int memcg_low_reclaim:1;
	unsigned int memcg_low_skipped:1;
108

109 110 111 112 113 114 115 116 117 118
	unsigned int hibernation_mode:1;

	/* One of the zones is ready for compaction */
	unsigned int compaction_ready:1;

	/* Incremented by the number of inactive pages that were scanned */
	unsigned long nr_scanned;

	/* Number of pages freed so far during a call to shrink_zones() */
	unsigned long nr_reclaimed;
Linus Torvalds's avatar
Linus Torvalds committed
119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
};

#ifdef ARCH_HAS_PREFETCH
#define prefetch_prev_lru_page(_page, _base, _field)			\
	do {								\
		if ((_page)->lru.prev != _base) {			\
			struct page *prev;				\
									\
			prev = lru_to_page(&(_page->lru));		\
			prefetch(&prev->_field);			\
		}							\
	} while (0)
#else
#define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
#endif

#ifdef ARCH_HAS_PREFETCHW
#define prefetchw_prev_lru_page(_page, _base, _field)			\
	do {								\
		if ((_page)->lru.prev != _base) {			\
			struct page *prev;				\
									\
			prev = lru_to_page(&(_page->lru));		\
			prefetchw(&prev->_field);			\
		}							\
	} while (0)
#else
#define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
#endif

/*
 * From 0 .. 100.  Higher means more swappy.
 */
int vm_swappiness = 60;
153 154 155 156 157
/*
 * The total number of pages which are beyond the high watermark within all
 * zones.
 */
unsigned long vm_total_pages;
Linus Torvalds's avatar
Linus Torvalds committed
158 159 160 161

static LIST_HEAD(shrinker_list);
static DECLARE_RWSEM(shrinker_rwsem);

Andrew Morton's avatar
Andrew Morton committed
162
#ifdef CONFIG_MEMCG
163 164
static bool global_reclaim(struct scan_control *sc)
{
165
	return !sc->target_mem_cgroup;
166
}
167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187

/**
 * sane_reclaim - is the usual dirty throttling mechanism operational?
 * @sc: scan_control in question
 *
 * The normal page dirty throttling mechanism in balance_dirty_pages() is
 * completely broken with the legacy memcg and direct stalling in
 * shrink_page_list() is used for throttling instead, which lacks all the
 * niceties such as fairness, adaptive pausing, bandwidth proportional
 * allocation and configurability.
 *
 * This function tests whether the vmscan currently in progress can assume
 * that the normal dirty throttling mechanism is operational.
 */
static bool sane_reclaim(struct scan_control *sc)
{
	struct mem_cgroup *memcg = sc->target_mem_cgroup;

	if (!memcg)
		return true;
#ifdef CONFIG_CGROUP_WRITEBACK
188
	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
189 190 191 192
		return true;
#endif
	return false;
}
193
#else
194 195 196 197
static bool global_reclaim(struct scan_control *sc)
{
	return true;
}
198 199 200 201 202

static bool sane_reclaim(struct scan_control *sc)
{
	return true;
}
203 204
#endif

205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222
/*
 * This misses isolated pages which are not accounted for to save counters.
 * As the data only determines if reclaim or compaction continues, it is
 * not expected that isolated pages will be a dominating factor.
 */
unsigned long zone_reclaimable_pages(struct zone *zone)
{
	unsigned long nr;

	nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) +
		zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE);
	if (get_nr_swap_pages() > 0)
		nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) +
			zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON);

	return nr;
}

223 224 225 226 227 228 229
unsigned long pgdat_reclaimable_pages(struct pglist_data *pgdat)
{
	unsigned long nr;

	nr = node_page_state_snapshot(pgdat, NR_ACTIVE_FILE) +
	     node_page_state_snapshot(pgdat, NR_INACTIVE_FILE) +
	     node_page_state_snapshot(pgdat, NR_ISOLATED_FILE);
230 231

	if (get_nr_swap_pages() > 0)
232 233 234
		nr += node_page_state_snapshot(pgdat, NR_ACTIVE_ANON) +
		      node_page_state_snapshot(pgdat, NR_INACTIVE_ANON) +
		      node_page_state_snapshot(pgdat, NR_ISOLATED_ANON);
235 236 237 238

	return nr;
}

239 240 241 242 243 244 245
/**
 * lruvec_lru_size -  Returns the number of pages on the given LRU list.
 * @lruvec: lru vector
 * @lru: lru to use
 * @zone_idx: zones to consider (use MAX_NR_ZONES for the whole LRU list)
 */
unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx)
246
{
247 248 249
	unsigned long lru_size;
	int zid;

250
	if (!mem_cgroup_disabled())
251 252 253
		lru_size = mem_cgroup_get_lru_size(lruvec, lru);
	else
		lru_size = node_page_state(lruvec_pgdat(lruvec), NR_LRU_BASE + lru);
254

255 256 257
	for (zid = zone_idx + 1; zid < MAX_NR_ZONES; zid++) {
		struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];
		unsigned long size;
258

259 260 261 262 263 264 265 266 267 268 269 270
		if (!managed_zone(zone))
			continue;

		if (!mem_cgroup_disabled())
			size = mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
		else
			size = zone_page_state(&lruvec_pgdat(lruvec)->node_zones[zid],
				       NR_ZONE_LRU_BASE + lru);
		lru_size -= min(size, lru_size);
	}

	return lru_size;
271 272 273

}

Linus Torvalds's avatar
Linus Torvalds committed
274
/*
Glauber Costa's avatar
Glauber Costa committed
275
 * Add a shrinker callback to be called from the vm.
Linus Torvalds's avatar
Linus Torvalds committed
276
 */
Glauber Costa's avatar
Glauber Costa committed
277
int register_shrinker(struct shrinker *shrinker)
Linus Torvalds's avatar
Linus Torvalds committed
278
{
Glauber Costa's avatar
Glauber Costa committed
279 280 281 282 283 284 285 286 287
	size_t size = sizeof(*shrinker->nr_deferred);

	if (shrinker->flags & SHRINKER_NUMA_AWARE)
		size *= nr_node_ids;

	shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);
	if (!shrinker->nr_deferred)
		return -ENOMEM;

288 289 290
	down_write(&shrinker_rwsem);
	list_add_tail(&shrinker->list, &shrinker_list);
	up_write(&shrinker_rwsem);
Glauber Costa's avatar
Glauber Costa committed
291
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
292
}
293
EXPORT_SYMBOL(register_shrinker);
Linus Torvalds's avatar
Linus Torvalds committed
294 295 296 297

/*
 * Remove one
 */
298
void unregister_shrinker(struct shrinker *shrinker)
Linus Torvalds's avatar
Linus Torvalds committed
299 300 301 302
{
	down_write(&shrinker_rwsem);
	list_del(&shrinker->list);
	up_write(&shrinker_rwsem);
303
	kfree(shrinker->nr_deferred);
Linus Torvalds's avatar
Linus Torvalds committed
304
}
305
EXPORT_SYMBOL(unregister_shrinker);
Linus Torvalds's avatar
Linus Torvalds committed
306 307

#define SHRINK_BATCH 128
Glauber Costa's avatar
Glauber Costa committed
308

309 310 311 312
static unsigned long do_shrink_slab(struct shrink_control *shrinkctl,
				    struct shrinker *shrinker,
				    unsigned long nr_scanned,
				    unsigned long nr_eligible)
Glauber Costa's avatar
Glauber Costa committed
313 314 315 316
{
	unsigned long freed = 0;
	unsigned long long delta;
	long total_scan;
317
	long freeable;
Glauber Costa's avatar
Glauber Costa committed
318 319 320 321 322
	long nr;
	long new_nr;
	int nid = shrinkctl->nid;
	long batch_size = shrinker->batch ? shrinker->batch
					  : SHRINK_BATCH;
323
	long scanned = 0, next_deferred;
Glauber Costa's avatar
Glauber Costa committed
324

325 326
	freeable = shrinker->count_objects(shrinker, shrinkctl);
	if (freeable == 0)
Glauber Costa's avatar
Glauber Costa committed
327 328 329 330 331 332 333 334 335 336
		return 0;

	/*
	 * copy the current shrinker scan count into a local variable
	 * and zero it so that other concurrent shrinker invocations
	 * don't also do this scanning work.
	 */
	nr = atomic_long_xchg(&shrinker->nr_deferred[nid], 0);

	total_scan = nr;
337
	delta = (4 * nr_scanned) / shrinker->seeks;
338
	delta *= freeable;
339
	do_div(delta, nr_eligible + 1);
Glauber Costa's avatar
Glauber Costa committed
340 341
	total_scan += delta;
	if (total_scan < 0) {
342
		pr_err("shrink_slab: %pF negative objects to delete nr=%ld\n",
343
		       shrinker->scan_objects, total_scan);
344
		total_scan = freeable;
345 346 347
		next_deferred = nr;
	} else
		next_deferred = total_scan;
Glauber Costa's avatar
Glauber Costa committed
348 349 350 351 352 353 354

	/*
	 * We need to avoid excessive windup on filesystem shrinkers
	 * due to large numbers of GFP_NOFS allocations causing the
	 * shrinkers to return -1 all the time. This results in a large
	 * nr being built up so when a shrink that can do some work
	 * comes along it empties the entire cache due to nr >>>
355
	 * freeable. This is bad for sustaining a working set in
Glauber Costa's avatar
Glauber Costa committed
356 357 358 359 360
	 * memory.
	 *
	 * Hence only allow the shrinker to scan the entire cache when
	 * a large delta change is calculated directly.
	 */
361 362
	if (delta < freeable / 4)
		total_scan = min(total_scan, freeable / 2);
Glauber Costa's avatar
Glauber Costa committed
363 364 365 366 367 368

	/*
	 * Avoid risking looping forever due to too large nr value:
	 * never try to free more than twice the estimate number of
	 * freeable entries.
	 */
369 370
	if (total_scan > freeable * 2)
		total_scan = freeable * 2;
Glauber Costa's avatar
Glauber Costa committed
371 372

	trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,
373 374
				   nr_scanned, nr_eligible,
				   freeable, delta, total_scan);
Glauber Costa's avatar
Glauber Costa committed
375

376 377 378 379 380 381 382 383 384 385 386
	/*
	 * Normally, we should not scan less than batch_size objects in one
	 * pass to avoid too frequent shrinker calls, but if the slab has less
	 * than batch_size objects in total and we are really tight on memory,
	 * we will try to reclaim all available objects, otherwise we can end
	 * up failing allocations although there are plenty of reclaimable
	 * objects spread over several slabs with usage less than the
	 * batch_size.
	 *
	 * We detect the "tight on memory" situations by looking at the total
	 * number of objects we want to scan (total_scan). If it is greater
387
	 * than the total number of objects on slab (freeable), we must be
388 389 390 391
	 * scanning at high prio and therefore should try to reclaim as much as
	 * possible.
	 */
	while (total_scan >= batch_size ||
392
	       total_scan >= freeable) {
393
		unsigned long ret;
394
		unsigned long nr_to_scan = min(batch_size, total_scan);
Glauber Costa's avatar
Glauber Costa committed
395

396
		shrinkctl->nr_to_scan = nr_to_scan;
397
		shrinkctl->nr_scanned = nr_to_scan;
398 399 400 401
		ret = shrinker->scan_objects(shrinker, shrinkctl);
		if (ret == SHRINK_STOP)
			break;
		freed += ret;
Glauber Costa's avatar
Glauber Costa committed
402

403 404 405
		count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned);
		total_scan -= shrinkctl->nr_scanned;
		scanned += shrinkctl->nr_scanned;
Glauber Costa's avatar
Glauber Costa committed
406 407 408 409

		cond_resched();
	}

410 411 412 413
	if (next_deferred >= scanned)
		next_deferred -= scanned;
	else
		next_deferred = 0;
Glauber Costa's avatar
Glauber Costa committed
414 415 416 417 418
	/*
	 * move the unused scan count back into the shrinker in a
	 * manner that handles concurrent updates. If we exhausted the
	 * scan, there is no need to do an update.
	 */
419 420
	if (next_deferred > 0)
		new_nr = atomic_long_add_return(next_deferred,
Glauber Costa's avatar
Glauber Costa committed
421 422 423 424
						&shrinker->nr_deferred[nid]);
	else
		new_nr = atomic_long_read(&shrinker->nr_deferred[nid]);

425
	trace_mm_shrink_slab_end(shrinker, nid, freed, nr, new_nr, total_scan);
Glauber Costa's avatar
Glauber Costa committed
426
	return freed;
427 428
}

429
/**
430
 * shrink_slab - shrink slab caches
431 432
 * @gfp_mask: allocation context
 * @nid: node whose slab caches to target
433
 * @memcg: memory cgroup whose slab caches to target
434 435
 * @nr_scanned: pressure numerator
 * @nr_eligible: pressure denominator
Linus Torvalds's avatar
Linus Torvalds committed
436
 *
437
 * Call the shrink functions to age shrinkable caches.
Linus Torvalds's avatar
Linus Torvalds committed
438
 *
439 440
 * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set,
 * unaware shrinkers will receive a node id of 0 instead.
Linus Torvalds's avatar
Linus Torvalds committed
441
 *
442 443
 * @memcg specifies the memory cgroup to target. If it is not NULL,
 * only shrinkers with SHRINKER_MEMCG_AWARE set will be called to scan
444 445
 * objects from the memory cgroup specified. Otherwise, only unaware
 * shrinkers are called.
446
 *
447 448 449 450 451 452 453
 * @nr_scanned and @nr_eligible form a ratio that indicate how much of
 * the available objects should be scanned.  Page reclaim for example
 * passes the number of pages scanned and the number of pages on the
 * LRU lists that it considered on @nid, plus a bias in @nr_scanned
 * when it encountered mapped pages.  The ratio is further biased by
 * the ->seeks setting of the shrink function, which indicates the
 * cost to recreate an object relative to that of an LRU page.
454
 *
455
 * Returns the number of reclaimed slab objects.
Linus Torvalds's avatar
Linus Torvalds committed
456
 */
457 458 459 460
static unsigned long shrink_slab(gfp_t gfp_mask, int nid,
				 struct mem_cgroup *memcg,
				 unsigned long nr_scanned,
				 unsigned long nr_eligible)
Linus Torvalds's avatar
Linus Torvalds committed
461 462
{
	struct shrinker *shrinker;
Dave Chinner's avatar
Dave Chinner committed
463
	unsigned long freed = 0;
Linus Torvalds's avatar
Linus Torvalds committed
464

465
	if (memcg && (!memcg_kmem_enabled() || !mem_cgroup_online(memcg)))
466 467
		return 0;

468 469
	if (nr_scanned == 0)
		nr_scanned = SWAP_CLUSTER_MAX;
Linus Torvalds's avatar
Linus Torvalds committed
470

471
	if (!down_read_trylock(&shrinker_rwsem)) {
Dave Chinner's avatar
Dave Chinner committed
472 473 474 475 476 477 478
		/*
		 * If we would return 0, our callers would understand that we
		 * have nothing else to shrink and give up trying. By returning
		 * 1 we keep it going and assume we'll be able to shrink next
		 * time.
		 */
		freed = 1;
479 480
		goto out;
	}
Linus Torvalds's avatar
Linus Torvalds committed
481 482

	list_for_each_entry(shrinker, &shrinker_list, list) {
483 484 485
		struct shrink_control sc = {
			.gfp_mask = gfp_mask,
			.nid = nid,
486
			.memcg = memcg,
487
		};
488

489 490 491 492 493 494 495
		/*
		 * If kernel memory accounting is disabled, we ignore
		 * SHRINKER_MEMCG_AWARE flag and call all shrinkers
		 * passing NULL for memcg.
		 */
		if (memcg_kmem_enabled() &&
		    !!memcg != !!(shrinker->flags & SHRINKER_MEMCG_AWARE))
496 497
			continue;

498 499
		if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
			sc.nid = 0;
Linus Torvalds's avatar
Linus Torvalds committed
500

501
		freed += do_shrink_slab(&sc, shrinker, nr_scanned, nr_eligible);
Linus Torvalds's avatar
Linus Torvalds committed
502
	}
503

Linus Torvalds's avatar
Linus Torvalds committed
504
	up_read(&shrinker_rwsem);
505 506
out:
	cond_resched();
Dave Chinner's avatar
Dave Chinner committed
507
	return freed;
Linus Torvalds's avatar
Linus Torvalds committed
508 509
}

510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532
void drop_slab_node(int nid)
{
	unsigned long freed;

	do {
		struct mem_cgroup *memcg = NULL;

		freed = 0;
		do {
			freed += shrink_slab(GFP_KERNEL, nid, memcg,
					     1000, 1000);
		} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
	} while (freed > 10);
}

void drop_slab(void)
{
	int nid;

	for_each_online_node(nid)
		drop_slab_node(nid);
}

Linus Torvalds's avatar
Linus Torvalds committed
533 534
static inline int is_page_cache_freeable(struct page *page)
{
535 536 537 538 539
	/*
	 * A freeable page cache page is referenced only by the caller
	 * that isolated the page, the page cache radix tree and
	 * optional buffer heads at page->private.
	 */
540 541 542
	int radix_pins = PageTransHuge(page) && PageSwapCache(page) ?
		HPAGE_PMD_NR : 1;
	return page_count(page) - page_has_private(page) == 1 + radix_pins;
Linus Torvalds's avatar
Linus Torvalds committed
543 544
}

545
static int may_write_to_inode(struct inode *inode, struct scan_control *sc)
Linus Torvalds's avatar
Linus Torvalds committed
546
{
547
	if (current->flags & PF_SWAPWRITE)
Linus Torvalds's avatar
Linus Torvalds committed
548
		return 1;
549
	if (!inode_write_congested(inode))
Linus Torvalds's avatar
Linus Torvalds committed
550
		return 1;
551
	if (inode_to_bdi(inode) == current->backing_dev_info)
Linus Torvalds's avatar
Linus Torvalds committed
552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570
		return 1;
	return 0;
}

/*
 * We detected a synchronous write error writing a page out.  Probably
 * -ENOSPC.  We need to propagate that into the address_space for a subsequent
 * fsync(), msync() or close().
 *
 * The tricky part is that after writepage we cannot touch the mapping: nothing
 * prevents it from being freed up.  But we have a ref on the page and once
 * that page is locked, the mapping is pinned.
 *
 * We're allowed to run sleeping lock_page() here because we know the caller has
 * __GFP_FS.
 */
static void handle_write_error(struct address_space *mapping,
				struct page *page, int error)
{
Jens Axboe's avatar
Jens Axboe committed
571
	lock_page(page);
572 573
	if (page_mapping(page) == mapping)
		mapping_set_error(mapping, error);
Linus Torvalds's avatar
Linus Torvalds committed
574 575 576
	unlock_page(page);
}

577 578 579 580 581 582 583 584 585 586 587 588
/* possible outcome of pageout() */
typedef enum {
	/* failed to write page out, page is locked */
	PAGE_KEEP,
	/* move page to the active list, page is locked */
	PAGE_ACTIVATE,
	/* page has been sent to the disk successfully, page is unlocked */
	PAGE_SUCCESS,
	/* page is clean and locked */
	PAGE_CLEAN,
} pageout_t;

Linus Torvalds's avatar
Linus Torvalds committed
589
/*
590 591
 * pageout is called by shrink_page_list() for each dirty page.
 * Calls ->writepage().
Linus Torvalds's avatar
Linus Torvalds committed
592
 */
593
static pageout_t pageout(struct page *page, struct address_space *mapping,
594
			 struct scan_control *sc)
Linus Torvalds's avatar
Linus Torvalds committed
595 596 597 598 599 600 601 602
{
	/*
	 * If the page is dirty, only perform writeback if that write
	 * will be non-blocking.  To prevent this allocation from being
	 * stalled by pagecache activity.  But note that there may be
	 * stalls if we need to run get_block().  We could test
	 * PagePrivate for that.
	 *
603
	 * If this process is currently in __generic_file_write_iter() against
Linus Torvalds's avatar
Linus Torvalds committed
604 605 606 607 608 609 610 611 612 613 614 615 616 617 618
	 * this page's queue, we can perform writeback even if that
	 * will block.
	 *
	 * If the page is swapcache, write it back even if that would
	 * block, for some throttling. This happens by accident, because
	 * swap_backing_dev_info is bust: it doesn't reflect the
	 * congestion state of the swapdevs.  Easy to fix, if needed.
	 */
	if (!is_page_cache_freeable(page))
		return PAGE_KEEP;
	if (!mapping) {
		/*
		 * Some data journaling orphaned pages can have
		 * page->mapping == NULL while being dirty with clean buffers.
		 */
619
		if (page_has_private(page)) {
Linus Torvalds's avatar
Linus Torvalds committed
620 621
			if (try_to_free_buffers(page)) {
				ClearPageDirty(page);
622
				pr_info("%s: orphaned page\n", __func__);
Linus Torvalds's avatar
Linus Torvalds committed
623 624 625 626 627 628 629
				return PAGE_CLEAN;
			}
		}
		return PAGE_KEEP;
	}
	if (mapping->a_ops->writepage == NULL)
		return PAGE_ACTIVATE;
630
	if (!may_write_to_inode(mapping->host, sc))
Linus Torvalds's avatar
Linus Torvalds committed
631 632 633 634 635 636 637
		return PAGE_KEEP;

	if (clear_page_dirty_for_io(page)) {
		int res;
		struct writeback_control wbc = {
			.sync_mode = WB_SYNC_NONE,
			.nr_to_write = SWAP_CLUSTER_MAX,
638 639
			.range_start = 0,
			.range_end = LLONG_MAX,
Linus Torvalds's avatar
Linus Torvalds committed
640 641 642 643 644 645 646
			.for_reclaim = 1,
		};

		SetPageReclaim(page);
		res = mapping->a_ops->writepage(page, &wbc);
		if (res < 0)
			handle_write_error(mapping, page, res);
647
		if (res == AOP_WRITEPAGE_ACTIVATE) {
Linus Torvalds's avatar
Linus Torvalds committed
648 649 650
			ClearPageReclaim(page);
			return PAGE_ACTIVATE;
		}
651

Linus Torvalds's avatar
Linus Torvalds committed
652 653 654 655
		if (!PageWriteback(page)) {
			/* synchronous write or broken a_ops? */
			ClearPageReclaim(page);
		}
656
		trace_mm_vmscan_writepage(page);
657
		inc_node_page_state(page, NR_VMSCAN_WRITE);
Linus Torvalds's avatar
Linus Torvalds committed
658 659 660 661 662 663
		return PAGE_SUCCESS;
	}

	return PAGE_CLEAN;
}

664
/*
Nick Piggin's avatar
Nick Piggin committed
665 666
 * Same as remove_mapping, but if the page is removed from the mapping, it
 * gets returned with a refcount of 0.
667
 */
668 669
static int __remove_mapping(struct address_space *mapping, struct page *page,
			    bool reclaimed)
670
{
671
	unsigned long flags;
672
	int refcount;
673

674 675
	BUG_ON(!PageLocked(page));
	BUG_ON(mapping != page_mapping(page));
676

677
	spin_lock_irqsave(&mapping->tree_lock, flags);
678
	/*
679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697
	 * The non racy check for a busy page.
	 *
	 * Must be careful with the order of the tests. When someone has
	 * a ref to the page, it may be possible that they dirty it then
	 * drop the reference. So if PageDirty is tested before page_count
	 * here, then the following race may occur:
	 *
	 * get_user_pages(&page);
	 * [user mapping goes away]
	 * write_to(page);
	 *				!PageDirty(page)    [good]
	 * SetPageDirty(page);
	 * put_page(page);
	 *				!page_count(page)   [good, discard it]
	 *
	 * [oops, our write_to data is lost]
	 *
	 * Reversing the order of the tests ensures such a situation cannot
	 * escape unnoticed. The smp_rmb is needed to ensure the page->flags
698
	 * load is not satisfied before that of page->_refcount.
699 700 701
	 *
	 * Note that if SetPageDirty is always performed via set_page_dirty,
	 * and thus under tree_lock, then this ordering is not required.
702
	 */
703 704 705 706 707
	if (unlikely(PageTransHuge(page)) && PageSwapCache(page))
		refcount = 1 + HPAGE_PMD_NR;
	else
		refcount = 2;
	if (!page_ref_freeze(page, refcount))
708
		goto cannot_free;
Nick Piggin's avatar
Nick Piggin committed
709 710
	/* note: atomic_cmpxchg in page_freeze_refs provides the smp_rmb */
	if (unlikely(PageDirty(page))) {
711
		page_ref_unfreeze(page, refcount);
712
		goto cannot_free;
Nick Piggin's avatar
Nick Piggin committed
713
	}
714 715 716

	if (PageSwapCache(page)) {
		swp_entry_t swap = { .val = page_private(page) };
717
		mem_cgroup_swapout(page, swap);
718
		__delete_from_swap_cache(page);
719
		spin_unlock_irqrestore(&mapping->tree_lock, flags);
720
		put_swap_page(page, swap);
Nick Piggin's avatar
Nick Piggin committed
721
	} else {
722
		void (*freepage)(struct page *);
723
		void *shadow = NULL;
724 725

		freepage = mapping->a_ops->freepage;
726 727 728 729 730 731 732 733 734
		/*
		 * Remember a shadow entry for reclaimed file cache in
		 * order to detect refaults, thus thrashing, later on.
		 *
		 * But don't store shadows in an address space that is
		 * already exiting.  This is not just an optizimation,
		 * inode reclaim needs to empty out the radix tree or
		 * the nodes are lost.  Don't plant shadows behind its
		 * back.
735 736 737 738 739 740
		 *
		 * We also don't store shadows for DAX mappings because the
		 * only page cache pages found in these are zero pages
		 * covering holes, and because we don't want to mix DAX
		 * exceptional entries and shadow exceptional entries in the
		 * same page_tree.
741 742
		 */
		if (reclaimed && page_is_file_cache(page) &&
743
		    !mapping_exiting(mapping) && !dax_mapping(mapping))
744
			shadow = workingset_eviction(mapping, page);
745
		__delete_from_page_cache(page, shadow);
746
		spin_unlock_irqrestore(&mapping->tree_lock, flags);
747 748 749

		if (freepage != NULL)
			freepage(page);
750 751 752 753 754
	}

	return 1;

cannot_free:
755
	spin_unlock_irqrestore(&mapping->tree_lock, flags);
756 757 758
	return 0;
}

Nick Piggin's avatar
Nick Piggin committed
759 760 761 762 763 764 765 766
/*
 * Attempt to detach a locked page from its ->mapping.  If it is dirty or if
 * someone else has a ref on the page, abort and return 0.  If it was
 * successfully detached, return 1.  Assumes the caller has a single ref on
 * this page.
 */
int remove_mapping(struct address_space *mapping, struct page *page)
{
767
	if (__remove_mapping(mapping, page, false)) {
Nick Piggin's avatar
Nick Piggin committed
768 769 770 771 772
		/*
		 * Unfreezing the refcount with 1 rather than 2 effectively
		 * drops the pagecache ref for us without requiring another
		 * atomic operation.
		 */
773
		page_ref_unfreeze(page, 1);
Nick Piggin's avatar
Nick Piggin committed
774 775 776 777 778
		return 1;
	}
	return 0;
}

779 780 781 782 783 784 785 786 787 788 789
/**
 * putback_lru_page - put previously isolated page onto appropriate LRU list
 * @page: page to be put back to appropriate lru list
 *
 * Add previously isolated @page to appropriate LRU list.
 * Page may still be unevictable for other reasons.
 *
 * lru_lock must not be held, interrupts must be enabled.
 */
void putback_lru_page(struct page *page)
{
790
	bool is_unevictable;
791
	int was_unevictable = PageUnevictable(page);
792

793
	VM_BUG_ON_PAGE(PageLRU(page), page);
794 795 796 797

redo:
	ClearPageUnevictable(page);

798
	if (page_evictable(page)) {
799 800 801 802 803 804
		/*
		 * For evictable pages, we can use the cache.
		 * In event of a race, worst case is we end up with an
		 * unevictable page on [in]active list.
		 * We know how to handle that.
		 */
805
		is_unevictable = false;
806
		lru_cache_add(page);
807 808 809 810 811
	} else {
		/*
		 * Put unevictable pages directly on zone's unevictable
		 * list.
		 */
812
		is_unevictable = true;
813
		add_page_to_unevictable_list(page);
814
		/*
815 816 817
		 * When racing with an mlock or AS_UNEVICTABLE clearing
		 * (page is unlocked) make sure that if the other thread
		 * does not observe our setting of PG_lru and fails
818
		 * isolation/check_move_unevictable_pages,
819
		 * we see PG_mlocked/AS_UNEVICTABLE cleared below and move
820 821
		 * the page back to the evictable list.
		 *
822
		 * The other side is TestClearPageMlocked() or shmem_lock().
823 824
		 */
		smp_mb();
825 826 827 828 829 830 831
	}

	/*
	 * page's status can change while we move it among lru. If an evictable
	 * page is on unevictable list, it never be freed. To avoid that,
	 * check after we added it to the list, again.
	 */
832
	if (is_unevictable && page_evictable(page)) {
833 834 835 836 837 838 839 840 841 842
		if (!isolate_lru_page(page)) {
			put_page(page);
			goto redo;
		}
		/* This means someone else dropped this page from LRU
		 * So, it will be freed or putback to LRU again. There is
		 * nothing to do here.
		 */
	}

843
	if (was_unevictable && !is_unevictable)
844
		count_vm_event(UNEVICTABLE_PGRESCUED);
845
	else if (!was_unevictable && is_unevictable)
846 847
		count_vm_event(UNEVICTABLE_PGCULLED);

848 849 850
	put_page(page);		/* drop ref from isolate */
}

851 852 853
enum page_references {
	PAGEREF_RECLAIM,
	PAGEREF_RECLAIM_CLEAN,
854
	PAGEREF_KEEP,
855 856 857 858 859 860
	PAGEREF_ACTIVATE,
};

static enum page_references page_check_references(struct page *page,
						  struct scan_control *sc)
{
861
	int referenced_ptes, referenced_page;
862 863
	unsigned long vm_flags;

864 865
	referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup,
					  &vm_flags);
866
	referenced_page = TestClearPageReferenced(page);
867 868 869 870 871 872 873 874

	/*
	 * Mlock lost the isolation race with us.  Let try_to_unmap()
	 * move the page to the unevictable list.
	 */
	if (vm_flags & VM_LOCKED)
		return PAGEREF_RECLAIM;

875
	if (referenced_ptes) {
876
		if (PageSwapBacked(page))
877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893
			return PAGEREF_ACTIVATE;
		/*
		 * All mapped pages start out with page table
		 * references from the instantiating fault, so we need
		 * to look twice if a mapped file page is used more
		 * than once.
		 *
		 * Mark it and spare it for another trip around the
		 * inactive list.  Another page table reference will
		 * lead to its activation.
		 *
		 * Note: the mark is set for activated pages as well
		 * so that recently deactivated but used pages are
		 * quickly recovered.
		 */
		SetPageReferenced(page);

894
		if (referenced_page || referenced_ptes > 1)
895 896
			return PAGEREF_ACTIVATE;

897 898 899 900 901 902
		/*
		 * Activate file-backed executable pages after first usage.
		 */
		if (vm_flags & VM_EXEC)
			return PAGEREF_ACTIVATE;

903 904
		return PAGEREF_KEEP;
	}
905 906

	/* Reclaim if clean, defer dirty pages to writeback */
907
	if (referenced_page && !PageSwapBacked(page))
908 909 910
		return PAGEREF_RECLAIM_CLEAN;

	return PAGEREF_RECLAIM;
911 912
}

913 914 915 916
/* Check if a page is dirty or under writeback */
static void page_check_dirty_writeback(struct page *page,
				       bool *dirty, bool *writeback)
{
917 918
	struct address_space *mapping;

919 920 921 922
	/*
	 * Anonymous pages are not handled by flushers and must be written
	 * from reclaim context. Do not stall reclaim based on them
	 */
Shaohua Li's avatar
Shaohua Li committed
923 924
	if (!page_is_file_cache(page) ||
	    (PageAnon(page) && !PageSwapBacked(page))) {
925 926 927 928 929 930 931 932
		*dirty = false;
		*writeback = false;
		return;
	}

	/* By default assume that the page flags are accurate */
	*dirty = PageDirty(page);
	*writeback = PageWriteback(page);
933 934 935 936 937 938 939 940

	/* Verify dirty/writeback state if the filesystem supports it */
	if (!page_has_private(page))
		return;

	mapping = page_mapping(page);
	if (mapping && mapping->a_ops->is_dirty_writeback)
		mapping->a_ops->is_dirty_writeback(page, dirty, writeback);
941 942
}

943 944 945 946 947 948
struct reclaim_stat {
	unsigned nr_dirty;
	unsigned nr_unqueued_dirty;
	unsigned nr_congested;
	unsigned nr_writeback;
	unsigned nr_immediate;
949 950 951
	unsigned nr_activate;
	unsigned nr_ref_keep;
	unsigned nr_unmap_fail;
952 953
};

Linus Torvalds's avatar
Linus Torvalds committed
954
/*
955
 * shrink_page_list() returns the number of reclaimed pages
Linus Torvalds's avatar
Linus Torvalds committed
956
 */
957
static unsigned long shrink_page_list(struct list_head *page_list,
958
				      struct pglist_data *pgdat,
959
				      struct scan_control *sc,
960
				      enum ttu_flags ttu_flags,
961
				      struct reclaim_stat *stat,
962
				      bool force_reclaim)
Linus Torvalds's avatar
Linus Torvalds committed
963 964
{
	LIST_HEAD(ret_pages);
965
	LIST_HEAD(free_pages);
Linus Torvalds's avatar
Linus Torvalds committed
966
	int pgactivate = 0;
967 968 969 970 971 972
	unsigned nr_unqueued_dirty = 0;
	unsigned nr_dirty = 0;
	unsigned nr_congested = 0;
	unsigned nr_reclaimed = 0;
	unsigned nr_writeback = 0;
	unsigned nr_immediate = 0;
973 974
	unsigned nr_ref_keep = 0;
	unsigned nr_unmap_fail = 0;
Linus Torvalds's avatar
Linus Torvalds committed
975 976 977 978 979 980 981

	cond_resched();

	while (!list_empty(page_list)) {
		struct address_space *mapping;
		struct page *page;
		int may_enter_fs;
982
		enum page_references references = PAGEREF_RECLAIM_CLEAN;
983
		bool dirty, writeback;
Linus Torvalds's avatar
Linus Torvalds committed
984 985 986 987 988 989

		cond_resched();

		page = lru_to_page(page_list);
		list_del(&page->lru);

Nick Piggin's avatar
Nick Piggin committed
990
		if (!trylock_page(page))
Linus Torvalds's avatar
Linus Torvalds committed
991 992
			goto keep;

993
		VM_BUG_ON_PAGE(PageActive(page), page);
Linus Torvalds's avatar
Linus Torvalds committed
994 995

		sc->nr_scanned++;
996

997
		if (unlikely(!page_evictable(page)))
Minchan Kim's avatar
Minchan Kim committed
998
			goto activate_locked;
999

1000
		if (!sc->may_unmap && page_mapped(page))
1001 1002
			goto keep_locked;

Linus Torvalds's avatar
Linus Torvalds committed
1003
		/* Double the slab pressure for mapped and swapcache pages */
Shaohua Li's avatar
Shaohua Li committed
1004 1005
		if ((page_mapped(page) || PageSwapCache(page)) &&
		    !(PageAnon(page) && !PageSwapBacked(page)))
Linus Torvalds's avatar
Linus Torvalds committed
1006 1007
			sc->nr_scanned++;

1008 1009 1010
		may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
			(PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));

1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023
		/*
		 * The number of dirty pages determines if a zone is marked
		 * reclaim_congested which affects wait_iff_congested. kswapd
		 * will stall and start writing pages if the tail of the LRU
		 * is all dirty unqueued pages.
		 */
		page_check_dirty_writeback(page, &dirty, &writeback);
		if (dirty || writeback)
			nr_dirty++;

		if (dirty && !writeback)
			nr_unqueued_dirty++;

1024 1025 1026 1027 1028 1029
		/*
		 * Treat this page as congested if the underlying BDI is or if
		 * pages are cycling through the LRU so quickly that the
		 * pages marked for immediate reclaim are making it to the
		 * end of the LRU a second time.
		 */
1030
		mapping = page_mapping(page);
1031
		if (((dirty || writeback) && mapping &&
1032
		     inode_write_congested(mapping->host)) ||
1033
		    (writeback && PageReclaim(page)))
1034 1035
			nr_congested++;

1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046
		/*
		 * If a page at the tail of the LRU is under writeback, there
		 * are three cases to consider.
		 *
		 * 1) If reclaim is encountering an excessive number of pages
		 *    under writeback and this page is both under writeback and
		 *    PageReclaim then it indicates that pages are being queued
		 *    for IO but are being recycled through the LRU before the
		 *    IO can complete. Waiting on the page itself risks an
		 *    indefinite stall if it is impossible to writeback the
		 *    page due to IO error or disconnected storage so instead
1047 1048
		 *    note that the LRU is being scanned too quickly and the
		 *    caller can stall after page list has been processed.
1049
		 *
1050
		 * 2) Global or new memcg reclaim encounters a page that is
1051 1052 1053
		 *    not marked for immediate reclaim, or the caller does not
		 *    have __GFP_FS (or __GFP_IO if it's simply going to swap,
		 *    not to fs). In this case mark the page for immediate
1054
		 *    reclaim and continue scanning.
1055
		 *
1056 1057
		 *    Require may_enter_fs because we would wait on fs, which
		 *    may not have submitted IO yet. And the loop driver might
1058 1059 1060 1061 1062
		 *    enter reclaim, and deadlock if it waits on a page for
		 *    which it is needed to do the write (loop masks off
		 *    __GFP_IO|__GFP_FS for this reason); but more thought
		 *    would probably show more reasons.
		 *
1063
		 * 3) Legacy memcg encounters a page that is already marked
1064 1065 1066 1067
		 *    PageReclaim. memcg does not have any dirty pages
		 *    throttling so we could easily OOM just because too many
		 *    pages are in writeback and there is nothing else to
		 *    reclaim. Wait for the writeback to complete.
1068 1069 1070 1071 1072 1073 1074 1075 1076
		 *
		 * In cases 1) and 2) we activate the pages to get them out of
		 * the way while we continue scanning for clean pages on the
		 * inactive list and refilling from the active list. The
		 * observation here is that waiting for disk writes is more
		 * expensive than potentially causing reloads down the line.
		 * Since they're marked for immediate reclaim, they won't put
		 * memory pressure on the cache working set any longer than it
		 * takes to write them to disk.
1077
		 */
1078
		if (PageWriteback(page)) {
1079 1080 1081
			/* Case 1 above */
			if (current_is_kswapd() &&
			    PageReclaim(page) &&
1082
			    test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {
1083
				nr_immediate++;
1084
				goto activate_locked;
1085 1086

			/* Case 2 above */
1087
			} else if (sane_reclaim(sc) ||
1088
			    !PageReclaim(page) || !may_enter_fs) {
1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
				/*
				 * This is slightly racy - end_page_writeback()
				 * might have just cleared PageReclaim, then
				 * setting PageReclaim here end up interpreted
				 * as PageReadahead - but that does not matter
				 * enough to care.  What we do want is for this
				 * page to have PageReclaim set next time memcg
				 * reclaim reaches the tests above, so it will
				 * then wait_on_page_writeback() to avoid OOM;
				 * and it's also appropriate in global reclaim.
				 */
				SetPageReclaim(page);
1101
				nr_writeback++;
1102
				goto activate_locked;
1103 1104 1105

			/* Case 3 above */
			} else {
1106
				unlock_page(page);
1107
				wait_on_page_writeback(page);
1108 1109 1110
				/* then go back and try same page again */
				list_add_tail(&page->lru, page_list);
				continue;
1111
			}
1112
		}
Linus Torvalds's avatar
Linus Torvalds committed
1113

1114 1115 1116
		if (!force_reclaim)
			references = page_check_references(page, sc);

1117 1118
		switch (references) {
		case PAGEREF_ACTIVATE:
Linus Torvalds's avatar
Linus Torvalds committed
1119
			goto activate_locked;
1120
		case PAGEREF_KEEP:
1121
			nr_ref_keep++;
1122
			goto keep_locked;
1123 1124 1125 1126
		case PAGEREF_RECLAIM:
		case PAGEREF_RECLAIM_CLEAN:
			; /* try to reclaim the page below */
		}
Linus Torvalds's avatar
Linus Torvalds committed
1127 1128 1129 1130

		/*
		 * Anonymous process memory has backing store?
		 * Try to allocate it some swap space here.
Shaohua Li's avatar
Shaohua Li committed
1131
		 * Lazyfree page could be freed directly
Linus Torvalds's avatar
Linus Torvalds committed
1132
		 */
1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
		if (PageAnon(page) && PageSwapBacked(page)) {
			if (!PageSwapCache(page)) {
				if (!(sc->gfp_mask & __GFP_IO))
					goto keep_locked;
				if (PageTransHuge(page)) {
					/* cannot split THP, skip it */
					if (!can_split_huge_page(page, NULL))
						goto activate_locked;
					/*
					 * Split pages without a PMD map right
					 * away. Chances are some or all of the
					 * tail pages can be freed without IO.
					 */
					if (!compound_mapcount(page) &&
					    split_huge_page_to_list(page,
								    page_list))
						goto activate_locked;
				}
				if (!add_to_swap(page)) {
					if (!PageTransHuge(page))
						goto activate_locked;
					/* Fallback to swap normal pages */
					if (split_huge_page_to_list(page,
								    page_list))
						goto activate_locked;
1158 1159 1160
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
					count_vm_event(THP_SWPOUT_FALLBACK);
#endif
1161 1162 1163
					if (!add_to_swap(page))
						goto activate_locked;
				}
1164

1165
				may_enter_fs = 1;
Linus Torvalds's avatar
Linus Torvalds committed
1166

1167 1168 1169
				/* Adding to swap updated mapping */
				mapping = page_mapping(page);
			}
1170 1171 1172 1173
		} else if (unlikely(PageTransHuge(page))) {
			/* Split file THP */
			if (split_huge_page_to_list(page, page_list))
				goto keep_locked;
1174
		}
Linus Torvalds's avatar
Linus Torvalds committed
1175 1176 1177 1178 1179

		/*
		 * The page is mapped into the page tables of one or more
		 * processes. Try to unmap it here.
		 */
Shaohua Li's avatar
Shaohua Li committed
1180
		if (page_mapped(page)) {
1181 1182 1183 1184 1185
			enum ttu_flags flags = ttu_flags | TTU_BATCH_FLUSH;

			if (unlikely(PageTransHuge(page)))
				flags |= TTU_SPLIT_HUGE_PMD;
			if (!try_to_unmap(page, flags)) {
1186
				nr_unmap_fail++;
Linus Torvalds's avatar
Linus Torvalds committed
1187 1188 1189 1190 1191
				goto activate_locked;
			}
		}

		if (PageDirty(page)) {
1192
			/*
1193 1194 1195 1196 1197 1198 1199 1200
			 * Only kswapd can writeback filesystem pages
			 * to avoid risk of stack overflow. But avoid
			 * injecting inefficient single-page IO into
			 * flusher writeback as much as possible: only
			 * write pages when we've encountered many
			 * dirty pages, and when we've already scanned
			 * the rest of the LRU for clean pages and see
			 * the same dirty pages again (PageReclaim).
1201
			 */
1202
			if (page_is_file_cache(page) &&
1203 1204
			    (!current_is_kswapd() || !PageReclaim(page) ||
			     !test_bit(PGDAT_DIRTY, &pgdat->flags))) {
1205 1206 1207 1208 1209 1210
				/*
				 * Immediately reclaim when written back.
				 * Similar in principal to deactivate_page()
				 * except we already have the page isolated
				 * and know it's dirty
				 */
1211
				inc_node_page_state(page, NR_VMSCAN_IMMEDIATE);
1212 1213
				SetPageReclaim(page);

1214
				goto activate_locked;
1215 1216
			}

1217
			if (references == PAGEREF_RECLAIM_CLEAN)
Linus Torvalds's avatar
Linus Torvalds committed
1218
				goto keep_locked;
1219
			if (!may_enter_fs)
Linus Torvalds's avatar
Linus Torvalds committed
1220
				goto keep_locked;
1221
			if (!sc->may_writepage)
Linus Torvalds's avatar
Linus Torvalds committed
1222 1223
				goto keep_locked;

1224 1225 1226 1227 1228 1229
			/*
			 * Page is dirty. Flush the TLB if a writable entry
			 * potentially exists to avoid CPU writes after IO
			 * starts and then write it out here.
			 */
			try_to_unmap_flush_dirty();
1230
			switch (pageout(page, mapping, sc)) {
Linus Torvalds's avatar
Linus Torvalds committed
1231 1232 1233 1234 1235
			case PAGE_KEEP:
				goto keep_locked;
			case PAGE_ACTIVATE:
				goto activate_locked;
			case PAGE_SUCCESS:
1236
				if (PageWriteback(page))
1237
					goto keep;
1238
				if (PageDirty(page))
Linus Torvalds's avatar
Linus Torvalds committed
1239
					goto keep;
1240

Linus Torvalds's avatar
Linus Torvalds committed
1241 1242 1243 1244
				/*
				 * A synchronous write - probably a ramdisk.  Go
				 * ahead and try to reclaim the page.
				 */
Nick Piggin's avatar
Nick Piggin committed
1245
				if (!trylock_page(page))
Linus Torvalds's avatar
Linus Torvalds committed
1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264
					goto keep;
				if (PageDirty(page) || PageWriteback(page))
					goto keep_locked;
				mapping = page_mapping(page);
			case PAGE_CLEAN:
				; /* try to free the page below */
			}
		}

		/*
		 * If the page has buffers, try to free the buffer mappings
		 * associated with this page. If we succeed we try to free
		 * the page as well.
		 *
		 * We do this even if the page is PageDirty().
		 * try_to_release_page() does not perform I/O, but it is
		 * possible for a page to have PageDirty set, but it is actually
		 * clean (all its buffers are clean).  This happens if the
		 * buffers were written out directly, with submit_bh(). ext3
1265
		 * will do this, as well as the blockdev mapping.
Linus Torvalds's avatar
Linus Torvalds committed
1266 1267 1268 1269 1270 1271 1272 1273 1274 1275
		 * try_to_release_page() will discover that cleanness and will
		 * drop the buffers and mark the page clean - it can be freed.
		 *
		 * Rarely, pages can have buffers and no ->mapping.  These are
		 * the pages which were not successfully invalidated in
		 * truncate_complete_page().  We try to drop those buffers here
		 * and if that worked, and the page is no longer mapped into
		 * process address space (page_count == 1) it can be freed.
		 * Otherwise, leave the page on the LRU so it is swappable.
		 */
1276
		if (page_has_private(page)) {
Linus Torvalds's avatar
Linus Torvalds committed
1277 1278
			if (!try_to_release_page(page, sc->gfp_mask))
				goto activate_locked;
Nick Piggin's avatar
Nick Piggin committed
1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294
			if (!mapping && page_count(page) == 1) {
				unlock_page(page);
				if (put_page_testzero(page))
					goto free_it;
				else {
					/*
					 * rare race with speculative reference.
					 * the speculative reference will free
					 * this page shortly, so we may
					 * increment nr_reclaimed here (and
					 * leave it off the LRU).
					 */
					nr_reclaimed++;
					continue;
				}
			}
Linus Torvalds's avatar
Linus Torvalds committed
1295 1296
		}

Shaohua Li's avatar
Shaohua Li committed
1297 1298 1299 1300 1301 1302 1303 1304
		if (PageAnon(page) && !PageSwapBacked(page)) {
			/* follow __remove_mapping for reference */
			if (!page_ref_freeze(page, 1))
				goto keep_locked;
			if (PageDirty(page)) {
				page_ref_unfreeze(page, 1);
				goto keep_locked;
			}
Linus Torvalds's avatar
Linus Torvalds committed
1305

Shaohua Li's avatar
Shaohua Li committed
1306
			count_vm_event(PGLAZYFREED);
1307
			count_memcg_page_event(page, PGLAZYFREED);
Shaohua Li's avatar
Shaohua Li committed
1308 1309
		} else if (!mapping || !__remove_mapping(mapping, page, true))
			goto keep_locked;
Nick Piggin's avatar
Nick Piggin committed
1310 1311 1312 1313 1314 1315 1316
		/*
		 * At this point, we have no other references and there is
		 * no way to pick any more up (removed from LRU, removed
		 * from pagecache). Can use non-atomic bitops now (and
		 * we obviously don't have to worry about waking up a process
		 * waiting on the page lock, because there are no references.
		 */
<