backing-dev.c 27.3 KB
Newer Older
1 2 3

#include <linux/wait.h>
#include <linux/backing-dev.h>
4 5
#include <linux/kthread.h>
#include <linux/freezer.h>
6
#include <linux/fs.h>
7
#include <linux/pagemap.h>
8
#include <linux/mm.h>
9 10
#include <linux/sched.h>
#include <linux/module.h>
11 12
#include <linux/writeback.h>
#include <linux/device.h>
13
#include <trace/events/writeback.h>
14

Jörn Engel's avatar
Jörn Engel committed
15 16
struct backing_dev_info noop_backing_dev_info = {
	.name		= "noop",
17
	.capabilities	= BDI_CAP_NO_ACCT_AND_WRITEBACK,
Jörn Engel's avatar
Jörn Engel committed
18
};
Tejun Heo's avatar
Tejun Heo committed
19
EXPORT_SYMBOL_GPL(noop_backing_dev_info);
Jörn Engel's avatar
Jörn Engel committed
20

21
static struct class *bdi_class;
22 23

/*
24
 * bdi_lock protects updates to bdi_list. bdi_list has RCU reader side
25 26
 * locking.
 */
27
DEFINE_SPINLOCK(bdi_lock);
28
LIST_HEAD(bdi_list);
29

30 31 32
/* bdi_wq serves all asynchronous writeback tasks */
struct workqueue_struct *bdi_wq;

33 34 35 36 37 38 39 40 41 42 43 44 45 46
#ifdef CONFIG_DEBUG_FS
#include <linux/debugfs.h>
#include <linux/seq_file.h>

static struct dentry *bdi_debug_root;

static void bdi_debug_init(void)
{
	bdi_debug_root = debugfs_create_dir("bdi", NULL);
}

static int bdi_debug_stats_show(struct seq_file *m, void *v)
{
	struct backing_dev_info *bdi = m->private;
Christoph Hellwig's avatar
Christoph Hellwig committed
47
	struct bdi_writeback *wb = &bdi->wb;
48 49
	unsigned long background_thresh;
	unsigned long dirty_thresh;
50
	unsigned long wb_thresh;
51
	unsigned long nr_dirty, nr_io, nr_more_io, nr_dirty_time;
52 53
	struct inode *inode;

54
	nr_dirty = nr_io = nr_more_io = nr_dirty_time = 0;
55
	spin_lock(&wb->list_lock);
56
	list_for_each_entry(inode, &wb->b_dirty, i_io_list)
Christoph Hellwig's avatar
Christoph Hellwig committed
57
		nr_dirty++;
58
	list_for_each_entry(inode, &wb->b_io, i_io_list)
Christoph Hellwig's avatar
Christoph Hellwig committed
59
		nr_io++;
60
	list_for_each_entry(inode, &wb->b_more_io, i_io_list)
Christoph Hellwig's avatar
Christoph Hellwig committed
61
		nr_more_io++;
62
	list_for_each_entry(inode, &wb->b_dirty_time, i_io_list)
63 64
		if (inode->i_state & I_DIRTY_TIME)
			nr_dirty_time++;
65
	spin_unlock(&wb->list_lock);
66

67
	global_dirty_limits(&background_thresh, &dirty_thresh);
68
	wb_thresh = wb_calc_thresh(wb, dirty_thresh);
69 70 71

#define K(x) ((x) << (PAGE_SHIFT - 10))
	seq_printf(m,
72 73 74 75 76
		   "BdiWriteback:       %10lu kB\n"
		   "BdiReclaimable:     %10lu kB\n"
		   "BdiDirtyThresh:     %10lu kB\n"
		   "DirtyThresh:        %10lu kB\n"
		   "BackgroundThresh:   %10lu kB\n"
77
		   "BdiDirtied:         %10lu kB\n"
78 79 80 81 82
		   "BdiWritten:         %10lu kB\n"
		   "BdiWriteBandwidth:  %10lu kBps\n"
		   "b_dirty:            %10lu\n"
		   "b_io:               %10lu\n"
		   "b_more_io:          %10lu\n"
83
		   "b_dirty_time:       %10lu\n"
84 85
		   "bdi_list:           %10u\n"
		   "state:              %10lx\n",
86 87
		   (unsigned long) K(wb_stat(wb, WB_WRITEBACK)),
		   (unsigned long) K(wb_stat(wb, WB_RECLAIMABLE)),
88
		   K(wb_thresh),
89 90
		   K(dirty_thresh),
		   K(background_thresh),
91 92
		   (unsigned long) K(wb_stat(wb, WB_DIRTIED)),
		   (unsigned long) K(wb_stat(wb, WB_WRITTEN)),
93
		   (unsigned long) K(wb->write_bandwidth),
94 95 96
		   nr_dirty,
		   nr_io,
		   nr_more_io,
97
		   nr_dirty_time,
98
		   !list_empty(&bdi->bdi_list), bdi->wb.state);
99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
#undef K

	return 0;
}

static int bdi_debug_stats_open(struct inode *inode, struct file *file)
{
	return single_open(file, bdi_debug_stats_show, inode->i_private);
}

static const struct file_operations bdi_debug_stats_fops = {
	.open		= bdi_debug_stats_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
};

116
static int bdi_debug_register(struct backing_dev_info *bdi, const char *name)
117
{
118 119 120
	if (!bdi_debug_root)
		return -ENOMEM;

121
	bdi->debug_dir = debugfs_create_dir(name, bdi_debug_root);
122 123 124
	if (!bdi->debug_dir)
		return -ENOMEM;

125 126
	bdi->debug_stats = debugfs_create_file("stats", 0444, bdi->debug_dir,
					       bdi, &bdi_debug_stats_fops);
127 128 129 130 131 132
	if (!bdi->debug_stats) {
		debugfs_remove(bdi->debug_dir);
		return -ENOMEM;
	}

	return 0;
133 134 135 136 137 138 139 140 141 142 143
}

static void bdi_debug_unregister(struct backing_dev_info *bdi)
{
	debugfs_remove(bdi->debug_stats);
	debugfs_remove(bdi->debug_dir);
}
#else
static inline void bdi_debug_init(void)
{
}
144
static inline int bdi_debug_register(struct backing_dev_info *bdi,
145 146
				      const char *name)
{
147
	return 0;
148 149 150 151 152 153
}
static inline void bdi_debug_unregister(struct backing_dev_info *bdi)
{
}
#endif

154 155 156 157 158 159
static ssize_t read_ahead_kb_store(struct device *dev,
				  struct device_attribute *attr,
				  const char *buf, size_t count)
{
	struct backing_dev_info *bdi = dev_get_drvdata(dev);
	unsigned long read_ahead_kb;
160
	ssize_t ret;
161

162 163 164 165 166 167 168
	ret = kstrtoul(buf, 10, &read_ahead_kb);
	if (ret < 0)
		return ret;

	bdi->ra_pages = read_ahead_kb >> (PAGE_SHIFT - 10);

	return count;
169 170 171 172 173 174 175 176 177 178 179
}

#define K(pages) ((pages) << (PAGE_SHIFT - 10))

#define BDI_SHOW(name, expr)						\
static ssize_t name##_show(struct device *dev,				\
			   struct device_attribute *attr, char *page)	\
{									\
	struct backing_dev_info *bdi = dev_get_drvdata(dev);		\
									\
	return snprintf(page, PAGE_SIZE-1, "%lld\n", (long long)expr);	\
180 181
}									\
static DEVICE_ATTR_RW(name);
182 183 184

BDI_SHOW(read_ahead_kb, K(bdi->ra_pages))

185 186 187 188 189
static ssize_t min_ratio_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t count)
{
	struct backing_dev_info *bdi = dev_get_drvdata(dev);
	unsigned int ratio;
190 191 192 193 194 195 196 197 198
	ssize_t ret;

	ret = kstrtouint(buf, 10, &ratio);
	if (ret < 0)
		return ret;

	ret = bdi_set_min_ratio(bdi, ratio);
	if (!ret)
		ret = count;
199 200 201 202 203

	return ret;
}
BDI_SHOW(min_ratio, bdi->min_ratio)

204 205 206 207 208
static ssize_t max_ratio_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t count)
{
	struct backing_dev_info *bdi = dev_get_drvdata(dev);
	unsigned int ratio;
209 210 211 212 213 214 215 216 217
	ssize_t ret;

	ret = kstrtouint(buf, 10, &ratio);
	if (ret < 0)
		return ret;

	ret = bdi_set_max_ratio(bdi, ratio);
	if (!ret)
		ret = count;
218 219 220 221 222

	return ret;
}
BDI_SHOW(max_ratio, bdi->max_ratio)

223 224 225 226 227 228 229 230 231
static ssize_t stable_pages_required_show(struct device *dev,
					  struct device_attribute *attr,
					  char *page)
{
	struct backing_dev_info *bdi = dev_get_drvdata(dev);

	return snprintf(page, PAGE_SIZE-1, "%d\n",
			bdi_cap_stable_pages_required(bdi) ? 1 : 0);
}
232 233 234 235 236 237 238 239
static DEVICE_ATTR_RO(stable_pages_required);

static struct attribute *bdi_dev_attrs[] = {
	&dev_attr_read_ahead_kb.attr,
	&dev_attr_min_ratio.attr,
	&dev_attr_max_ratio.attr,
	&dev_attr_stable_pages_required.attr,
	NULL,
240
};
241
ATTRIBUTE_GROUPS(bdi_dev);
242 243 244 245

static __init int bdi_class_init(void)
{
	bdi_class = class_create(THIS_MODULE, "bdi");
246 247 248
	if (IS_ERR(bdi_class))
		return PTR_ERR(bdi_class);

249
	bdi_class->dev_groups = bdi_dev_groups;
250
	bdi_debug_init();
251

252 253
	return 0;
}
254
postcore_initcall(bdi_class_init);
255

Jan Kara's avatar
Jan Kara committed
256 257
static int bdi_init(struct backing_dev_info *bdi);

258 259 260 261
static int __init default_bdi_init(void)
{
	int err;

262
	bdi_wq = alloc_workqueue("writeback", WQ_MEM_RECLAIM | WQ_FREEZABLE |
263
					      WQ_UNBOUND | WQ_SYSFS, 0);
264 265 266
	if (!bdi_wq)
		return -ENOMEM;

267
	err = bdi_init(&noop_backing_dev_info);
268 269 270 271 272

	return err;
}
subsys_initcall(default_bdi_init);

273
/*
274
 * This function is used when the first inode for this wb is marked dirty. It
275 276 277 278 279 280 281 282
 * wakes-up the corresponding bdi thread which should then take care of the
 * periodic background write-out of dirty inodes. Since the write-out would
 * starts only 'dirty_writeback_interval' centisecs from now anyway, we just
 * set up a timer which wakes the bdi thread up later.
 *
 * Note, we wouldn't bother setting up the timer, but this function is on the
 * fast-path (used by '__mark_inode_dirty()'), so we save few context switches
 * by delaying the wake-up.
283 284 285
 *
 * We have to be careful not to postpone flush work if it is scheduled for
 * earlier. Thus we use queue_delayed_work().
286
 */
287
void wb_wakeup_delayed(struct bdi_writeback *wb)
288 289 290 291
{
	unsigned long timeout;

	timeout = msecs_to_jiffies(dirty_writeback_interval * 10);
292 293 294 295
	spin_lock_bh(&wb->work_lock);
	if (test_bit(WB_registered, &wb->state))
		queue_delayed_work(bdi_wq, &wb->dwork, timeout);
	spin_unlock_bh(&wb->work_lock);
296 297
}

298
/*
299
 * Initial write bandwidth: 100 MB/s
300
 */
301
#define INIT_BW		(100 << (20 - PAGE_SHIFT))
302

Tejun Heo's avatar
Tejun Heo committed
303
static int wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi,
304
		   int blkcg_id, gfp_t gfp)
305
{
306
	int i, err;
307

308
	memset(wb, 0, sizeof(*wb));
309

310 311
	if (wb != &bdi->wb)
		bdi_get(bdi);
312 313 314 315 316
	wb->bdi = bdi;
	wb->last_old_flush = jiffies;
	INIT_LIST_HEAD(&wb->b_dirty);
	INIT_LIST_HEAD(&wb->b_io);
	INIT_LIST_HEAD(&wb->b_more_io);
317
	INIT_LIST_HEAD(&wb->b_dirty_time);
318
	spin_lock_init(&wb->list_lock);
319

320 321 322 323 324
	wb->bw_time_stamp = jiffies;
	wb->balanced_dirty_ratelimit = INIT_BW;
	wb->dirty_ratelimit = INIT_BW;
	wb->write_bandwidth = INIT_BW;
	wb->avg_write_bandwidth = INIT_BW;
325

326 327 328
	spin_lock_init(&wb->work_lock);
	INIT_LIST_HEAD(&wb->work_list);
	INIT_DELAYED_WORK(&wb->dwork, wb_workfn);
329
	wb->dirty_sleep = jiffies;
330

331
	wb->congested = wb_congested_get_create(bdi, blkcg_id, gfp);
332 333 334 335
	if (!wb->congested) {
		err = -ENOMEM;
		goto out_put_bdi;
	}
336

Tejun Heo's avatar
Tejun Heo committed
337
	err = fprop_local_init_percpu(&wb->completions, gfp);
338
	if (err)
339
		goto out_put_cong;
340

341
	for (i = 0; i < NR_WB_STAT_ITEMS; i++) {
Tejun Heo's avatar
Tejun Heo committed
342
		err = percpu_counter_init(&wb->stat[i], 0, gfp);
343 344
		if (err)
			goto out_destroy_stat;
345
	}
346

347
	return 0;
348 349

out_destroy_stat:
350
	while (i--)
351 352 353 354
		percpu_counter_destroy(&wb->stat[i]);
	fprop_local_destroy_percpu(&wb->completions);
out_put_cong:
	wb_congested_put(wb->congested);
355 356 357
out_put_bdi:
	if (wb != &bdi->wb)
		bdi_put(bdi);
358
	return err;
359 360
}

361 362
static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb);

363 364 365
/*
 * Remove bdi from the global list and shutdown any threads we have running
 */
366
static void wb_shutdown(struct bdi_writeback *wb)
367
{
368
	/* Make sure nobody queues further work */
369 370 371
	spin_lock_bh(&wb->work_lock);
	if (!test_and_clear_bit(WB_registered, &wb->state)) {
		spin_unlock_bh(&wb->work_lock);
372 373 374 375 376 377
		/*
		 * Wait for wb shutdown to finish if someone else is just
		 * running wb_shutdown(). Otherwise we could proceed to wb /
		 * bdi destruction before wb_shutdown() is finished.
		 */
		wait_on_bit(&wb->state, WB_shutting_down, TASK_UNINTERRUPTIBLE);
378
		return;
379
	}
380
	set_bit(WB_shutting_down, &wb->state);
381
	spin_unlock_bh(&wb->work_lock);
382

383
	cgwb_remove_from_bdi_list(wb);
384
	/*
385 386 387
	 * Drain work list and shutdown the delayed_work.  !WB_registered
	 * tells wb_workfn() that @wb is dying and its work_list needs to
	 * be drained no matter what.
388
	 */
389 390 391
	mod_delayed_work(bdi_wq, &wb->dwork, 0);
	flush_delayed_work(&wb->dwork);
	WARN_ON(!list_empty(&wb->work_list));
392 393 394 395 396 397
	/*
	 * Make sure bit gets cleared after shutdown is finished. Matches with
	 * the barrier provided by test_and_clear_bit() above.
	 */
	smp_wmb();
	clear_bit(WB_shutting_down, &wb->state);
398 399
}

400
static void wb_exit(struct bdi_writeback *wb)
401 402 403 404 405 406 407
{
	int i;

	WARN_ON(delayed_work_pending(&wb->dwork));

	for (i = 0; i < NR_WB_STAT_ITEMS; i++)
		percpu_counter_destroy(&wb->stat[i]);
408

409
	fprop_local_destroy_percpu(&wb->completions);
410
	wb_congested_put(wb->congested);
411 412
	if (wb != &wb->bdi->wb)
		bdi_put(wb->bdi);
413
}
414

415 416 417 418 419 420 421
#ifdef CONFIG_CGROUP_WRITEBACK

#include <linux/memcontrol.h>

/*
 * cgwb_lock protects bdi->cgwb_tree, bdi->cgwb_congested_tree,
 * blkcg->cgwb_list, and memcg->cgwb_list.  bdi->cgwb_tree is also RCU
422
 * protected.
423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449
 */
static DEFINE_SPINLOCK(cgwb_lock);

/**
 * wb_congested_get_create - get or create a wb_congested
 * @bdi: associated bdi
 * @blkcg_id: ID of the associated blkcg
 * @gfp: allocation mask
 *
 * Look up the wb_congested for @blkcg_id on @bdi.  If missing, create one.
 * The returned wb_congested has its reference count incremented.  Returns
 * NULL on failure.
 */
struct bdi_writeback_congested *
wb_congested_get_create(struct backing_dev_info *bdi, int blkcg_id, gfp_t gfp)
{
	struct bdi_writeback_congested *new_congested = NULL, *congested;
	struct rb_node **node, *parent;
	unsigned long flags;
retry:
	spin_lock_irqsave(&cgwb_lock, flags);

	node = &bdi->cgwb_congested_tree.rb_node;
	parent = NULL;

	while (*node != NULL) {
		parent = *node;
450 451
		congested = rb_entry(parent, struct bdi_writeback_congested,
				     rb_node);
452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476
		if (congested->blkcg_id < blkcg_id)
			node = &parent->rb_left;
		else if (congested->blkcg_id > blkcg_id)
			node = &parent->rb_right;
		else
			goto found;
	}

	if (new_congested) {
		/* !found and storage for new one already allocated, insert */
		congested = new_congested;
		new_congested = NULL;
		rb_link_node(&congested->rb_node, parent, node);
		rb_insert_color(&congested->rb_node, &bdi->cgwb_congested_tree);
		goto found;
	}

	spin_unlock_irqrestore(&cgwb_lock, flags);

	/* allocate storage for new one and retry */
	new_congested = kzalloc(sizeof(*new_congested), gfp);
	if (!new_congested)
		return NULL;

	atomic_set(&new_congested->refcnt, 0);
477
	new_congested->__bdi = bdi;
478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503
	new_congested->blkcg_id = blkcg_id;
	goto retry;

found:
	atomic_inc(&congested->refcnt);
	spin_unlock_irqrestore(&cgwb_lock, flags);
	kfree(new_congested);
	return congested;
}

/**
 * wb_congested_put - put a wb_congested
 * @congested: wb_congested to put
 *
 * Put @congested and destroy it if the refcnt reaches zero.
 */
void wb_congested_put(struct bdi_writeback_congested *congested)
{
	unsigned long flags;

	local_irq_save(flags);
	if (!atomic_dec_and_lock(&congested->refcnt, &cgwb_lock)) {
		local_irq_restore(flags);
		return;
	}

504
	/* bdi might already have been destroyed leaving @congested unlinked */
505
	if (congested->__bdi) {
506
		rb_erase(&congested->rb_node,
507 508
			 &congested->__bdi->cgwb_congested_tree);
		congested->__bdi = NULL;
509 510
	}

511 512 513 514 515 516 517 518 519 520 521 522 523 524
	spin_unlock_irqrestore(&cgwb_lock, flags);
	kfree(congested);
}

static void cgwb_release_workfn(struct work_struct *work)
{
	struct bdi_writeback *wb = container_of(work, struct bdi_writeback,
						release_work);

	wb_shutdown(wb);

	css_put(wb->memcg_css);
	css_put(wb->blkcg_css);

525
	fprop_local_destroy_percpu(&wb->memcg_completions);
526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547
	percpu_ref_exit(&wb->refcnt);
	wb_exit(wb);
	kfree_rcu(wb, rcu);
}

static void cgwb_release(struct percpu_ref *refcnt)
{
	struct bdi_writeback *wb = container_of(refcnt, struct bdi_writeback,
						refcnt);
	schedule_work(&wb->release_work);
}

static void cgwb_kill(struct bdi_writeback *wb)
{
	lockdep_assert_held(&cgwb_lock);

	WARN_ON(!radix_tree_delete(&wb->bdi->cgwb_tree, wb->memcg_css->id));
	list_del(&wb->memcg_node);
	list_del(&wb->blkcg_node);
	percpu_ref_kill(&wb->refcnt);
}

548 549 550 551 552 553 554
static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb)
{
	spin_lock_irq(&cgwb_lock);
	list_del_rcu(&wb->bdi_node);
	spin_unlock_irq(&cgwb_lock);
}

555 556 557 558 559 560 561 562 563 564 565 566
static int cgwb_create(struct backing_dev_info *bdi,
		       struct cgroup_subsys_state *memcg_css, gfp_t gfp)
{
	struct mem_cgroup *memcg;
	struct cgroup_subsys_state *blkcg_css;
	struct blkcg *blkcg;
	struct list_head *memcg_cgwb_list, *blkcg_cgwb_list;
	struct bdi_writeback *wb;
	unsigned long flags;
	int ret = 0;

	memcg = mem_cgroup_from_css(memcg_css);
567
	blkcg_css = cgroup_get_e_css(memcg_css->cgroup, &io_cgrp_subsys);
568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584
	blkcg = css_to_blkcg(blkcg_css);
	memcg_cgwb_list = mem_cgroup_cgwb_list(memcg);
	blkcg_cgwb_list = &blkcg->cgwb_list;

	/* look up again under lock and discard on blkcg mismatch */
	spin_lock_irqsave(&cgwb_lock, flags);
	wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);
	if (wb && wb->blkcg_css != blkcg_css) {
		cgwb_kill(wb);
		wb = NULL;
	}
	spin_unlock_irqrestore(&cgwb_lock, flags);
	if (wb)
		goto out_put;

	/* need to create a new one */
	wb = kmalloc(sizeof(*wb), gfp);
585 586 587 588
	if (!wb) {
		ret = -ENOMEM;
		goto out_put;
	}
589

590
	ret = wb_init(wb, bdi, blkcg_css->id, gfp);
591 592 593 594 595 596 597
	if (ret)
		goto err_free;

	ret = percpu_ref_init(&wb->refcnt, cgwb_release, 0, gfp);
	if (ret)
		goto err_wb_exit;

598 599 600 601
	ret = fprop_local_init_percpu(&wb->memcg_completions, gfp);
	if (ret)
		goto err_ref_exit;

602 603 604 605
	wb->memcg_css = memcg_css;
	wb->blkcg_css = blkcg_css;
	INIT_WORK(&wb->release_work, cgwb_release_workfn);
	set_bit(WB_registered, &wb->state);
606 607

	/*
608 609 610 611
	 * The root wb determines the registered state of the whole bdi and
	 * memcg_cgwb_list and blkcg_cgwb_list's next pointers indicate
	 * whether they're still online.  Don't link @wb if any is dead.
	 * See wb_memcg_offline() and wb_blkcg_offline().
612
	 */
613 614 615 616 617 618 619
	ret = -ENODEV;
	spin_lock_irqsave(&cgwb_lock, flags);
	if (test_bit(WB_registered, &bdi->wb.state) &&
	    blkcg_cgwb_list->next && memcg_cgwb_list->next) {
		/* we might have raced another instance of this function */
		ret = radix_tree_insert(&bdi->cgwb_tree, memcg_css->id, wb);
		if (!ret) {
620
			list_add_tail_rcu(&wb->bdi_node, &bdi->wb_list);
621 622 623 624 625 626 627 628 629 630
			list_add(&wb->memcg_node, memcg_cgwb_list);
			list_add(&wb->blkcg_node, blkcg_cgwb_list);
			css_get(memcg_css);
			css_get(blkcg_css);
		}
	}
	spin_unlock_irqrestore(&cgwb_lock, flags);
	if (ret) {
		if (ret == -EEXIST)
			ret = 0;
631
		goto err_fprop_exit;
632 633 634
	}
	goto out_put;

635 636
err_fprop_exit:
	fprop_local_destroy_percpu(&wb->memcg_completions);
637 638 639 640 641 642 643 644 645
err_ref_exit:
	percpu_ref_exit(&wb->refcnt);
err_wb_exit:
	wb_exit(wb);
err_free:
	kfree(wb);
out_put:
	css_put(blkcg_css);
	return ret;
646 647
}

648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674
/**
 * wb_get_create - get wb for a given memcg, create if necessary
 * @bdi: target bdi
 * @memcg_css: cgroup_subsys_state of the target memcg (must have positive ref)
 * @gfp: allocation mask to use
 *
 * Try to get the wb for @memcg_css on @bdi.  If it doesn't exist, try to
 * create one.  The returned wb has its refcount incremented.
 *
 * This function uses css_get() on @memcg_css and thus expects its refcnt
 * to be positive on invocation.  IOW, rcu_read_lock() protection on
 * @memcg_css isn't enough.  try_get it before calling this function.
 *
 * A wb is keyed by its associated memcg.  As blkcg implicitly enables
 * memcg on the default hierarchy, memcg association is guaranteed to be
 * more specific (equal or descendant to the associated blkcg) and thus can
 * identify both the memcg and blkcg associations.
 *
 * Because the blkcg associated with a memcg may change as blkcg is enabled
 * and disabled closer to root in the hierarchy, each wb keeps track of
 * both the memcg and blkcg associated with it and verifies the blkcg on
 * each lookup.  On mismatch, the existing wb is discarded and a new one is
 * created.
 */
struct bdi_writeback *wb_get_create(struct backing_dev_info *bdi,
				    struct cgroup_subsys_state *memcg_css,
				    gfp_t gfp)
675
{
676 677
	struct bdi_writeback *wb;

678
	might_sleep_if(gfpflags_allow_blocking(gfp));
679 680 681 682 683 684 685 686 687 688 689 690

	if (!memcg_css->parent)
		return &bdi->wb;

	do {
		rcu_read_lock();
		wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);
		if (wb) {
			struct cgroup_subsys_state *blkcg_css;

			/* see whether the blkcg association has changed */
			blkcg_css = cgroup_get_e_css(memcg_css->cgroup,
691
						     &io_cgrp_subsys);
692 693 694 695 696 697 698 699 700 701
			if (unlikely(wb->blkcg_css != blkcg_css ||
				     !wb_tryget(wb)))
				wb = NULL;
			css_put(blkcg_css);
		}
		rcu_read_unlock();
	} while (!wb && !cgwb_create(bdi, memcg_css, gfp));

	return wb;
}
702

703
static int cgwb_bdi_init(struct backing_dev_info *bdi)
704
{
705 706
	int ret;

707 708
	INIT_RADIX_TREE(&bdi->cgwb_tree, GFP_ATOMIC);
	bdi->cgwb_congested_tree = RB_ROOT;
709 710 711

	ret = wb_init(&bdi->wb, bdi, 1, GFP_KERNEL);
	if (!ret) {
712
		bdi->wb.memcg_css = &root_mem_cgroup->css;
713 714 715
		bdi->wb.blkcg_css = blkcg_root_css;
	}
	return ret;
716 717
}

718
static void cgwb_bdi_unregister(struct backing_dev_info *bdi)
719 720 721
{
	struct radix_tree_iter iter;
	void **slot;
722
	struct bdi_writeback *wb;
723 724 725 726 727 728

	WARN_ON(test_bit(WB_registered, &bdi->wb.state));

	spin_lock_irq(&cgwb_lock);
	radix_tree_for_each_slot(slot, &bdi->cgwb_tree, &iter, 0)
		cgwb_kill(*slot);
729 730 731 732 733 734 735 736

	while (!list_empty(&bdi->wb_list)) {
		wb = list_first_entry(&bdi->wb_list, struct bdi_writeback,
				      bdi_node);
		spin_unlock_irq(&cgwb_lock);
		wb_shutdown(wb);
		spin_lock_irq(&cgwb_lock);
	}
737 738 739 740 741 742 743 744
	spin_unlock_irq(&cgwb_lock);
}

/**
 * wb_memcg_offline - kill all wb's associated with a memcg being offlined
 * @memcg: memcg being offlined
 *
 * Also prevents creation of any new wb's associated with @memcg.
745
 */
746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776
void wb_memcg_offline(struct mem_cgroup *memcg)
{
	LIST_HEAD(to_destroy);
	struct list_head *memcg_cgwb_list = mem_cgroup_cgwb_list(memcg);
	struct bdi_writeback *wb, *next;

	spin_lock_irq(&cgwb_lock);
	list_for_each_entry_safe(wb, next, memcg_cgwb_list, memcg_node)
		cgwb_kill(wb);
	memcg_cgwb_list->next = NULL;	/* prevent new wb's */
	spin_unlock_irq(&cgwb_lock);
}

/**
 * wb_blkcg_offline - kill all wb's associated with a blkcg being offlined
 * @blkcg: blkcg being offlined
 *
 * Also prevents creation of any new wb's associated with @blkcg.
 */
void wb_blkcg_offline(struct blkcg *blkcg)
{
	LIST_HEAD(to_destroy);
	struct bdi_writeback *wb, *next;

	spin_lock_irq(&cgwb_lock);
	list_for_each_entry_safe(wb, next, &blkcg->cgwb_list, blkcg_node)
		cgwb_kill(wb);
	blkcg->cgwb_list.next = NULL;	/* prevent new wb's */
	spin_unlock_irq(&cgwb_lock);
}

777 778 779 780 781 782 783 784 785 786
static void cgwb_bdi_exit(struct backing_dev_info *bdi)
{
	struct rb_node *rbn;

	spin_lock_irq(&cgwb_lock);
	while ((rbn = rb_first(&bdi->cgwb_congested_tree))) {
		struct bdi_writeback_congested *congested =
			rb_entry(rbn, struct bdi_writeback_congested, rb_node);

		rb_erase(rbn, &bdi->cgwb_congested_tree);
787
		congested->__bdi = NULL;	/* mark @congested unlinked */
788 789 790 791
	}
	spin_unlock_irq(&cgwb_lock);
}

792 793 794 795 796 797 798
static void cgwb_bdi_register(struct backing_dev_info *bdi)
{
	spin_lock_irq(&cgwb_lock);
	list_add_tail_rcu(&bdi->wb.bdi_node, &bdi->wb_list);
	spin_unlock_irq(&cgwb_lock);
}

799 800
#else	/* CONFIG_CGROUP_WRITEBACK */

801 802 803 804 805 806 807 808
static int cgwb_bdi_init(struct backing_dev_info *bdi)
{
	int err;

	bdi->wb_congested = kzalloc(sizeof(*bdi->wb_congested), GFP_KERNEL);
	if (!bdi->wb_congested)
		return -ENOMEM;

809 810
	atomic_set(&bdi->wb_congested->refcnt, 1);

811 812
	err = wb_init(&bdi->wb, bdi, 1, GFP_KERNEL);
	if (err) {
813
		wb_congested_put(bdi->wb_congested);
814 815 816 817 818
		return err;
	}
	return 0;
}

819
static void cgwb_bdi_unregister(struct backing_dev_info *bdi) { }
820 821

static void cgwb_bdi_exit(struct backing_dev_info *bdi)
822 823 824
{
	wb_congested_put(bdi->wb_congested);
}
825

826 827 828 829 830 831 832 833 834 835
static void cgwb_bdi_register(struct backing_dev_info *bdi)
{
	list_add_tail_rcu(&bdi->wb.bdi_node, &bdi->wb_list);
}

static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb)
{
	list_del_rcu(&wb->bdi_node);
}

836
#endif	/* CONFIG_CGROUP_WRITEBACK */
837

Jan Kara's avatar
Jan Kara committed
838
static int bdi_init(struct backing_dev_info *bdi)
839
{
840 841
	int ret;

842 843
	bdi->dev = NULL;

844
	kref_init(&bdi->refcnt);
845
	bdi->min_ratio = 0;
846
	bdi->max_ratio = 100;
847
	bdi->max_prop_frac = FPROP_FRAC_BASE;
848
	INIT_LIST_HEAD(&bdi->bdi_list);
849
	INIT_LIST_HEAD(&bdi->wb_list);
850
	init_waitqueue_head(&bdi->wb_waitq);
851

852 853 854
	ret = cgwb_bdi_init(bdi);

	return ret;
855
}
856

857 858 859 860 861 862 863 864 865 866 867 868 869 870 871
struct backing_dev_info *bdi_alloc_node(gfp_t gfp_mask, int node_id)
{
	struct backing_dev_info *bdi;

	bdi = kmalloc_node(sizeof(struct backing_dev_info),
			   gfp_mask | __GFP_ZERO, node_id);
	if (!bdi)
		return NULL;

	if (bdi_init(bdi)) {
		kfree(bdi);
		return NULL;
	}
	return bdi;
}
872
EXPORT_SYMBOL(bdi_alloc_node);
873

874
int bdi_register_va(struct backing_dev_info *bdi, const char *fmt, va_list args)
875 876
{
	struct device *dev;
877

878 879
	if (bdi->dev)	/* The driver needs to use separate queues per device */
		return 0;
880

881
	dev = device_create_vargs(bdi_class, NULL, MKDEV(0, 0), bdi, fmt, args);
882 883
	if (IS_ERR(dev))
		return PTR_ERR(dev);
884

885 886 887 888
	if (bdi_debug_register(bdi, dev_name(dev))) {
		device_destroy(bdi_class, dev->devt);
		return -ENOMEM;
	}
889
	cgwb_bdi_register(bdi);
890
	bdi->dev = dev;
891

892 893 894 895 896 897 898 899
	set_bit(WB_registered, &bdi->wb.state);

	spin_lock_bh(&bdi_lock);
	list_add_tail_rcu(&bdi->bdi_list, &bdi_list);
	spin_unlock_bh(&bdi_lock);

	trace_writeback_bdi_register(bdi);
	return 0;
900
}
901 902
EXPORT_SYMBOL(bdi_register_va);

903
int bdi_register(struct backing_dev_info *bdi, const char *fmt, ...)
904 905 906 907 908
{
	va_list args;
	int ret;

	va_start(args, fmt);
909
	ret = bdi_register_va(bdi, fmt, args);
910 911 912
	va_end(args);
	return ret;
}
913
EXPORT_SYMBOL(bdi_register);
914

915 916 917 918
int bdi_register_owner(struct backing_dev_info *bdi, struct device *owner)
{
	int rc;

919
	rc = bdi_register(bdi, "%u:%u", MAJOR(owner->devt), MINOR(owner->devt));
920 921
	if (rc)
		return rc;
Jan Kara's avatar
Jan Kara committed
922 923
	/* Leaking owner reference... */
	WARN_ON(bdi->owner);
924 925 926 927 928 929
	bdi->owner = owner;
	get_device(owner);
	return 0;
}
EXPORT_SYMBOL(bdi_register_owner);

930 931 932 933 934 935 936 937
/*
 * Remove bdi from bdi_list, and ensure that it is no longer visible
 */
static void bdi_remove_from_list(struct backing_dev_info *bdi)
{
	spin_lock_bh(&bdi_lock);
	list_del_rcu(&bdi->bdi_list);
	spin_unlock_bh(&bdi_lock);
938

939 940
	synchronize_rcu_expedited();
}
941

942
void bdi_unregister(struct backing_dev_info *bdi)
943
{
944 945 946
	/* make sure nobody finds us on the bdi_list anymore */
	bdi_remove_from_list(bdi);
	wb_shutdown(&bdi->wb);
947
	cgwb_bdi_unregister(bdi);
948

949 950 951 952 953
	if (bdi->dev) {
		bdi_debug_unregister(bdi);
		device_unregister(bdi->dev);
		bdi->dev = NULL;
	}
954 955 956 957 958

	if (bdi->owner) {
		put_device(bdi->owner);
		bdi->owner = NULL;
	}
959
}
960

961 962 963 964 965
static void release_bdi(struct kref *ref)
{
	struct backing_dev_info *bdi =
			container_of(ref, struct backing_dev_info, refcnt);

966 967
	if (test_bit(WB_registered, &bdi->wb.state))
		bdi_unregister(bdi);
Jan Kara's avatar
Jan Kara committed
968 969 970
	WARN_ON_ONCE(bdi->dev);
	wb_exit(&bdi->wb);
	cgwb_bdi_exit(bdi);
971 972 973 974 975 976 977
	kfree(bdi);
}

void bdi_put(struct backing_dev_info *bdi)
{
	kref_put(&bdi->refcnt, release_bdi);
}
978
EXPORT_SYMBOL(bdi_put);
979

980 981 982 983
static wait_queue_head_t congestion_wqh[2] = {
		__WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[0]),
		__WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1])
	};
984
static atomic_t nr_wb_congested[2];
985

986
void clear_wb_congested(struct bdi_writeback_congested *congested, int sync)
987
{
988
	wait_queue_head_t *wqh = &congestion_wqh[sync];
989
	enum wb_congested_state bit;
990

991
	bit = sync ? WB_sync_congested : WB_async_congested;
992 993
	if (test_and_clear_bit(bit, &congested->state))
		atomic_dec(&nr_wb_congested[sync]);
994
	smp_mb__after_atomic();
995 996 997
	if (waitqueue_active(wqh))
		wake_up(wqh);
}
998
EXPORT_SYMBOL(clear_wb_congested);
999

1000
void set_wb_congested(struct bdi_writeback_congested *congested, int sync)
1001
{
1002
	enum wb_congested_state bit;
1003

1004
	bit = sync ? WB_sync_congested : WB_async_congested;
1005 1006
	if (!test_and_set_bit(bit, &congested->state))
		atomic_inc(&nr_wb_congested[sync]);
1007
}
1008
EXPORT_SYMBOL(set_wb_congested);
1009 1010 1011

/**
 * congestion_wait - wait for a backing_dev to become uncongested
1012
 * @sync: SYNC or ASYNC IO
1013 1014 1015 1016 1017 1018
 * @timeout: timeout in jiffies
 *
 * Waits for up to @timeout jiffies for a backing_dev (any backing_dev) to exit
 * write congestion.  If no backing_devs are congested then just wait for the
 * next write to be completed.
 */
1019
long congestion_wait(int sync, long timeout)
1020 1021
{
	long ret;
1022
	unsigned long start = jiffies;
1023
	DEFINE_WAIT(wait);
1024
	wait_queue_head_t *wqh = &congestion_wqh[sync];
1025 1026 1027 1028

	prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
	ret = io_schedule_timeout(timeout);
	finish_wait(wqh, &wait);
1029 1030 1031 1032

	trace_writeback_congestion_wait(jiffies_to_usecs(timeout),
					jiffies_to_usecs(jiffies - start));

1033 1034 1035
	return ret;
}
EXPORT_SYMBOL(congestion_wait);
1036

1037
/**
1038 1039
 * wait_iff_congested - Conditionally wait for a backing_dev to become uncongested or a pgdat to complete writes
 * @pgdat: A pgdat to check if it is heavily congested
1040 1041 1042 1043
 * @sync: SYNC or ASYNC IO
 * @timeout: timeout in jiffies
 *
 * In the event of a congested backing_dev (any backing_dev) and the given
1044
 * @pgdat has experienced recent congestion, this waits for up to @timeout
1045 1046 1047
 * jiffies for either a BDI to exit congestion of the given @sync queue
 * or a write to complete.
 *
1048
 * In the absence of pgdat congestion, cond_resched() is called to yield
1049
 * the processor if necessary but otherwise does not sleep.
1050 1051 1052 1053 1054
 *
 * The return value is 0 if the sleep is for the full timeout. Otherwise,
 * it is the number of jiffies that were still remaining when the function
 * returned. return_value == timeout implies the function did not sleep.
 */
1055
long wait_iff_congested(struct pglist_data *pgdat, int sync, long timeout)
1056 1057 1058 1059 1060 1061 1062 1063
{
	long ret;
	unsigned long start = jiffies;
	DEFINE_WAIT(wait);
	wait_queue_head_t *wqh = &congestion_wqh[sync];

	/*
	 * If there is no congestion, or heavy congestion is not being
1064
	 * encountered in the current pgdat, yield if necessary instead
1065 1066
	 * of sleeping on the congestion queue
	 */
1067
	if (atomic_read(&nr_wb_congested[sync]) == 0 ||
1068
	    !test_bit(PGDAT_CONGESTED, &pgdat->flags)) {
1069
		cond_resched();
1070

1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090
		/* In case we scheduled, work out time remaining */
		ret = timeout - (jiffies - start);
		if (ret < 0)
			ret = 0;

		goto out;
	}

	/* Sleep until uncongested or a write happens */
	prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
	ret = io_schedule_timeout(timeout);
	finish_wait(wqh, &wait);

out:
	trace_writeback_wait_iff_congested(jiffies_to_usecs(timeout),
					jiffies_to_usecs(jiffies - start));

	return ret;
}
EXPORT_SYMBOL(wait_iff_congested);