block_dev.c 53.6 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12 13
/*
 *  linux/fs/block_dev.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *  Copyright (C) 2001  Andrea Arcangeli <andrea@suse.de> SuSE
 */

#include <linux/init.h>
#include <linux/mm.h>
#include <linux/fcntl.h>
#include <linux/slab.h>
#include <linux/kmod.h>
#include <linux/major.h>
14
#include <linux/device_cgroup.h>
Linus Torvalds's avatar
Linus Torvalds committed
15 16
#include <linux/highmem.h>
#include <linux/blkdev.h>
17
#include <linux/backing-dev.h>
Linus Torvalds's avatar
Linus Torvalds committed
18 19
#include <linux/module.h>
#include <linux/blkpg.h>
20
#include <linux/magic.h>
21
#include <linux/dax.h>
Linus Torvalds's avatar
Linus Torvalds committed
22
#include <linux/buffer_head.h>
Al Viro's avatar
Al Viro committed
23
#include <linux/swap.h>
24
#include <linux/pagevec.h>
25
#include <linux/writeback.h>
Linus Torvalds's avatar
Linus Torvalds committed
26 27 28 29
#include <linux/mpage.h>
#include <linux/mount.h>
#include <linux/uio.h>
#include <linux/namei.h>
30
#include <linux/log2.h>
Al Viro's avatar
Al Viro committed
31
#include <linux/cleancache.h>
32
#include <linux/dax.h>
33
#include <linux/badblocks.h>
34
#include <linux/task_io_accounting_ops.h>
35
#include <linux/falloc.h>
36
#include <linux/uaccess.h>
37
#include "internal.h"
Linus Torvalds's avatar
Linus Torvalds committed
38 39 40 41 42 43

struct bdev_inode {
	struct block_device bdev;
	struct inode vfs_inode;
};

Adrian Bunk's avatar
Adrian Bunk committed
44 45
static const struct address_space_operations def_blk_aops;

Linus Torvalds's avatar
Linus Torvalds committed
46 47 48 49 50
static inline struct bdev_inode *BDEV_I(struct inode *inode)
{
	return container_of(inode, struct bdev_inode, vfs_inode);
}

51
struct block_device *I_BDEV(struct inode *inode)
Linus Torvalds's avatar
Linus Torvalds committed
52 53 54 55 56
{
	return &BDEV_I(inode)->bdev;
}
EXPORT_SYMBOL(I_BDEV);

Toshi Kani's avatar
Toshi Kani committed
57 58 59 60 61 62 63 64 65 66 67 68
void __vfs_msg(struct super_block *sb, const char *prefix, const char *fmt, ...)
{
	struct va_format vaf;
	va_list args;

	va_start(args, fmt);
	vaf.fmt = fmt;
	vaf.va = &args;
	printk_ratelimited("%sVFS (%s): %pV\n", prefix, sb->s_id, &vaf);
	va_end(args);
}

69
static void bdev_write_inode(struct block_device *bdev)
70
{
71 72 73
	struct inode *inode = bdev->bd_inode;
	int ret;

74 75 76
	spin_lock(&inode->i_lock);
	while (inode->i_state & I_DIRTY) {
		spin_unlock(&inode->i_lock);
77 78 79 80 81 82 83
		ret = write_inode_now(inode, true);
		if (ret) {
			char name[BDEVNAME_SIZE];
			pr_warn_ratelimited("VFS: Dirty inode writeback failed "
					    "for block device %s (err=%d).\n",
					    bdevname(bdev, name), ret);
		}
84 85 86 87 88
		spin_lock(&inode->i_lock);
	}
	spin_unlock(&inode->i_lock);
}

Peter Zijlstra's avatar
Peter Zijlstra committed
89
/* Kill _all_ buffers and pagecache , dirty or not.. */
Al Viro's avatar
Al Viro committed
90
void kill_bdev(struct block_device *bdev)
Linus Torvalds's avatar
Linus Torvalds committed
91
{
Al Viro's avatar
Al Viro committed
92 93
	struct address_space *mapping = bdev->bd_inode->i_mapping;

94
	if (mapping->nrpages == 0 && mapping->nrexceptional == 0)
Peter Zijlstra's avatar
Peter Zijlstra committed
95
		return;
Al Viro's avatar
Al Viro committed
96

Peter Zijlstra's avatar
Peter Zijlstra committed
97
	invalidate_bh_lrus();
Al Viro's avatar
Al Viro committed
98
	truncate_inode_pages(mapping, 0);
Linus Torvalds's avatar
Linus Torvalds committed
99
}	
Al Viro's avatar
Al Viro committed
100 101 102 103 104 105 106
EXPORT_SYMBOL(kill_bdev);

/* Invalidate clean unused buffers and pagecache. */
void invalidate_bdev(struct block_device *bdev)
{
	struct address_space *mapping = bdev->bd_inode->i_mapping;

107 108 109 110 111
	if (mapping->nrpages) {
		invalidate_bh_lrus();
		lru_add_drain_all();	/* make sure all lru add caches are flushed */
		invalidate_mapping_pages(mapping, 0, -1);
	}
Al Viro's avatar
Al Viro committed
112 113 114
	/* 99% of the time, we don't need to flush the cleancache on the bdev.
	 * But, for the strange corners, lets be cautious
	 */
115
	cleancache_invalidate_inode(mapping);
Al Viro's avatar
Al Viro committed
116 117
}
EXPORT_SYMBOL(invalidate_bdev);
Linus Torvalds's avatar
Linus Torvalds committed
118

119 120 121 122 123 124 125 126 127 128 129 130 131 132
static void set_init_blocksize(struct block_device *bdev)
{
	unsigned bsize = bdev_logical_block_size(bdev);
	loff_t size = i_size_read(bdev->bd_inode);

	while (bsize < PAGE_SIZE) {
		if (size & bsize)
			break;
		bsize <<= 1;
	}
	bdev->bd_block_size = bsize;
	bdev->bd_inode->i_blkbits = blksize_bits(bsize);
}

Linus Torvalds's avatar
Linus Torvalds committed
133 134 135
int set_blocksize(struct block_device *bdev, int size)
{
	/* Size must be a power of two, and between 512 and PAGE_SIZE */
136
	if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size))
Linus Torvalds's avatar
Linus Torvalds committed
137 138 139
		return -EINVAL;

	/* Size cannot be smaller than the size supported by the device */
140
	if (size < bdev_logical_block_size(bdev))
Linus Torvalds's avatar
Linus Torvalds committed
141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
		return -EINVAL;

	/* Don't change the size if it is same as current */
	if (bdev->bd_block_size != size) {
		sync_blockdev(bdev);
		bdev->bd_block_size = size;
		bdev->bd_inode->i_blkbits = blksize_bits(size);
		kill_bdev(bdev);
	}
	return 0;
}

EXPORT_SYMBOL(set_blocksize);

int sb_set_blocksize(struct super_block *sb, int size)
{
	if (set_blocksize(sb->s_bdev, size))
		return 0;
	/* If we get here, we know size is power of two
	 * and it's value is between 512 and PAGE_SIZE */
	sb->s_blocksize = size;
162
	sb->s_blocksize_bits = blksize_bits(size);
Linus Torvalds's avatar
Linus Torvalds committed
163 164 165 166 167 168 169
	return sb->s_blocksize;
}

EXPORT_SYMBOL(sb_set_blocksize);

int sb_min_blocksize(struct super_block *sb, int size)
{
170
	int minsize = bdev_logical_block_size(sb->s_bdev);
Linus Torvalds's avatar
Linus Torvalds committed
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
	if (size < minsize)
		size = minsize;
	return sb_set_blocksize(sb, size);
}

EXPORT_SYMBOL(sb_min_blocksize);

static int
blkdev_get_block(struct inode *inode, sector_t iblock,
		struct buffer_head *bh, int create)
{
	bh->b_bdev = I_BDEV(inode);
	bh->b_blocknr = iblock;
	set_buffer_mapped(bh);
	return 0;
}

188 189 190 191 192
static struct inode *bdev_file_inode(struct file *file)
{
	return file->f_mapping->host;
}

193 194 195 196 197 198 199 200 201 202
static unsigned int dio_bio_write_op(struct kiocb *iocb)
{
	unsigned int op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;

	/* avoid the need for a I/O completion work item */
	if (iocb->ki_flags & IOCB_DSYNC)
		op |= REQ_FUA;
	return op;
}

203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
#define DIO_INLINE_BIO_VECS 4

static void blkdev_bio_end_io_simple(struct bio *bio)
{
	struct task_struct *waiter = bio->bi_private;

	WRITE_ONCE(bio->bi_private, NULL);
	wake_up_process(waiter);
}

static ssize_t
__blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
		int nr_pages)
{
	struct file *file = iocb->ki_filp;
	struct block_device *bdev = I_BDEV(bdev_file_inode(file));
219
	struct bio_vec inline_vecs[DIO_INLINE_BIO_VECS], *vecs, *bvec;
220 221 222 223 224 225 226
	loff_t pos = iocb->ki_pos;
	bool should_dirty = false;
	struct bio bio;
	ssize_t ret;
	blk_qc_t qc;
	int i;

227 228
	if ((pos | iov_iter_alignment(iter)) &
	    (bdev_logical_block_size(bdev) - 1))
229 230
		return -EINVAL;

231 232 233 234 235 236 237 238
	if (nr_pages <= DIO_INLINE_BIO_VECS)
		vecs = inline_vecs;
	else {
		vecs = kmalloc(nr_pages * sizeof(struct bio_vec), GFP_KERNEL);
		if (!vecs)
			return -ENOMEM;
	}

239
	bio_init(&bio, vecs, nr_pages);
240
	bio_set_dev(&bio, bdev);
241
	bio.bi_iter.bi_sector = pos >> 9;
242
	bio.bi_write_hint = iocb->ki_hint;
243 244 245 246 247
	bio.bi_private = current;
	bio.bi_end_io = blkdev_bio_end_io_simple;

	ret = bio_iov_iter_get_pages(&bio, iter);
	if (unlikely(ret))
248
		goto out;
249 250 251
	ret = bio.bi_iter.bi_size;

	if (iov_iter_rw(iter) == READ) {
252
		bio.bi_opf = REQ_OP_READ;
253 254 255
		if (iter_is_iovec(iter))
			should_dirty = true;
	} else {
256
		bio.bi_opf = dio_bio_write_op(iocb);
257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276
		task_io_account_write(ret);
	}

	qc = submit_bio(&bio);
	for (;;) {
		set_current_state(TASK_UNINTERRUPTIBLE);
		if (!READ_ONCE(bio.bi_private))
			break;
		if (!(iocb->ki_flags & IOCB_HIPRI) ||
		    !blk_mq_poll(bdev_get_queue(bdev), qc))
			io_schedule();
	}
	__set_current_state(TASK_RUNNING);

	bio_for_each_segment_all(bvec, &bio, i) {
		if (should_dirty && !PageCompound(bvec->bv_page))
			set_page_dirty_lock(bvec->bv_page);
		put_page(bvec->bv_page);
	}

277
	if (unlikely(bio.bi_status))
278
		ret = blk_status_to_errno(bio.bi_status);
279

280 281 282 283
out:
	if (vecs != inline_vecs)
		kfree(vecs);

284 285
	bio_uninit(&bio);

286 287 288
	return ret;
}

289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308
struct blkdev_dio {
	union {
		struct kiocb		*iocb;
		struct task_struct	*waiter;
	};
	size_t			size;
	atomic_t		ref;
	bool			multi_bio : 1;
	bool			should_dirty : 1;
	bool			is_sync : 1;
	struct bio		bio;
};

static struct bio_set *blkdev_dio_pool __read_mostly;

static void blkdev_bio_end_io(struct bio *bio)
{
	struct blkdev_dio *dio = bio->bi_private;
	bool should_dirty = dio->should_dirty;

309 310 311 312
	if (bio->bi_status && !dio->bio.bi_status)
		dio->bio.bi_status = bio->bi_status;

	if (!dio->multi_bio || atomic_dec_and_test(&dio->ref)) {
313 314
		if (!dio->is_sync) {
			struct kiocb *iocb = dio->iocb;
315
			ssize_t ret;
316

317
			if (likely(!dio->bio.bi_status)) {
318 319
				ret = dio->size;
				iocb->ki_pos += ret;
320 321
			} else {
				ret = blk_status_to_errno(dio->bio.bi_status);
322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345
			}

			dio->iocb->ki_complete(iocb, ret, 0);
			bio_put(&dio->bio);
		} else {
			struct task_struct *waiter = dio->waiter;

			WRITE_ONCE(dio->waiter, NULL);
			wake_up_process(waiter);
		}
	}

	if (should_dirty) {
		bio_check_pages_dirty(bio);
	} else {
		struct bio_vec *bvec;
		int i;

		bio_for_each_segment_all(bvec, bio, i)
			put_page(bvec->bv_page);
		bio_put(bio);
	}
}

346
static ssize_t
347
__blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
348 349
{
	struct file *file = iocb->ki_filp;
350
	struct inode *inode = bdev_file_inode(file);
351
	struct block_device *bdev = I_BDEV(inode);
352
	struct blk_plug plug;
353 354
	struct blkdev_dio *dio;
	struct bio *bio;
355
	bool is_read = (iov_iter_rw(iter) == READ), is_sync;
356 357
	loff_t pos = iocb->ki_pos;
	blk_qc_t qc = BLK_QC_T_NONE;
358
	int ret = 0;
359

360 361
	if ((pos | iov_iter_alignment(iter)) &
	    (bdev_logical_block_size(bdev) - 1))
362 363 364 365 366 367
		return -EINVAL;

	bio = bio_alloc_bioset(GFP_KERNEL, nr_pages, blkdev_dio_pool);
	bio_get(bio); /* extra ref for the completion handler */

	dio = container_of(bio, struct blkdev_dio, bio);
368
	dio->is_sync = is_sync = is_sync_kiocb(iocb);
369 370 371 372 373 374 375 376 377
	if (dio->is_sync)
		dio->waiter = current;
	else
		dio->iocb = iocb;

	dio->size = 0;
	dio->multi_bio = false;
	dio->should_dirty = is_read && (iter->type == ITER_IOVEC);

378
	blk_start_plug(&plug);
379
	for (;;) {
380
		bio_set_dev(bio, bdev);
381
		bio->bi_iter.bi_sector = pos >> 9;
382
		bio->bi_write_hint = iocb->ki_hint;
383 384 385 386 387
		bio->bi_private = dio;
		bio->bi_end_io = blkdev_bio_end_io;

		ret = bio_iov_iter_get_pages(bio, iter);
		if (unlikely(ret)) {
388
			bio->bi_status = BLK_STS_IOERR;
389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420
			bio_endio(bio);
			break;
		}

		if (is_read) {
			bio->bi_opf = REQ_OP_READ;
			if (dio->should_dirty)
				bio_set_pages_dirty(bio);
		} else {
			bio->bi_opf = dio_bio_write_op(iocb);
			task_io_account_write(bio->bi_iter.bi_size);
		}

		dio->size += bio->bi_iter.bi_size;
		pos += bio->bi_iter.bi_size;

		nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES);
		if (!nr_pages) {
			qc = submit_bio(bio);
			break;
		}

		if (!dio->multi_bio) {
			dio->multi_bio = true;
			atomic_set(&dio->ref, 2);
		} else {
			atomic_inc(&dio->ref);
		}

		submit_bio(bio);
		bio = bio_alloc(GFP_KERNEL, nr_pages);
	}
421
	blk_finish_plug(&plug);
422

423
	if (!is_sync)
424 425 426 427 428 429 430 431 432 433 434 435 436
		return -EIOCBQUEUED;

	for (;;) {
		set_current_state(TASK_UNINTERRUPTIBLE);
		if (!READ_ONCE(dio->waiter))
			break;

		if (!(iocb->ki_flags & IOCB_HIPRI) ||
		    !blk_mq_poll(bdev_get_queue(bdev), qc))
			io_schedule();
	}
	__set_current_state(TASK_RUNNING);

437
	if (!ret)
438
		ret = blk_status_to_errno(dio->bio.bi_status);
439
	if (likely(!ret))
440 441 442 443 444 445 446 447 448
		ret = dio->size;

	bio_put(&dio->bio);
	return ret;
}

static ssize_t
blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
{
449
	int nr_pages;
450

451
	nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES + 1);
452 453
	if (!nr_pages)
		return 0;
454
	if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_PAGES)
455
		return __blkdev_direct_IO_simple(iocb, iter, nr_pages);
456 457 458 459 460 461

	return __blkdev_direct_IO(iocb, iter, min(nr_pages, BIO_MAX_PAGES));
}

static __init int blkdev_init(void)
{
462
	blkdev_dio_pool = bioset_create(4, offsetof(struct blkdev_dio, bio), BIOSET_NEED_BVECS);
463 464 465
	if (!blkdev_dio_pool)
		return -ENOMEM;
	return 0;
466
}
467
module_init(blkdev_init);
468

469 470 471 472 473 474 475 476 477
int __sync_blockdev(struct block_device *bdev, int wait)
{
	if (!bdev)
		return 0;
	if (!wait)
		return filemap_flush(bdev->bd_inode->i_mapping);
	return filemap_write_and_wait(bdev->bd_inode->i_mapping);
}

478 479 480 481 482 483
/*
 * Write out and wait upon all the dirty data associated with a block
 * device via its mapping.  Does not take the superblock lock.
 */
int sync_blockdev(struct block_device *bdev)
{
484
	return __sync_blockdev(bdev, 1);
485 486 487 488 489 490 491 492 493 494 495 496
}
EXPORT_SYMBOL(sync_blockdev);

/*
 * Write out and wait upon all dirty data associated with this
 * device.   Filesystem data as well as the underlying block
 * device.  Takes the superblock lock.
 */
int fsync_bdev(struct block_device *bdev)
{
	struct super_block *sb = get_super(bdev);
	if (sb) {
497
		int res = sync_filesystem(sb);
498 499 500 501 502
		drop_super(sb);
		return res;
	}
	return sync_blockdev(bdev);
}
503
EXPORT_SYMBOL(fsync_bdev);
504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522

/**
 * freeze_bdev  --  lock a filesystem and force it into a consistent state
 * @bdev:	blockdevice to lock
 *
 * If a superblock is found on this device, we take the s_umount semaphore
 * on it to make sure nobody unmounts until the snapshot creation is done.
 * The reference counter (bd_fsfreeze_count) guarantees that only the last
 * unfreeze process can unfreeze the frozen filesystem actually when multiple
 * freeze requests arrive simultaneously. It counts up in freeze_bdev() and
 * count down in thaw_bdev(). When it becomes 0, thaw_bdev() will unfreeze
 * actually.
 */
struct super_block *freeze_bdev(struct block_device *bdev)
{
	struct super_block *sb;
	int error = 0;

	mutex_lock(&bdev->bd_fsfreeze_mutex);
523 524 525 526 527 528
	if (++bdev->bd_fsfreeze_count > 1) {
		/*
		 * We don't even need to grab a reference - the first call
		 * to freeze_bdev grab an active reference and only the last
		 * thaw_bdev drops it.
		 */
529
		sb = get_super(bdev);
530 531
		if (sb)
			drop_super(sb);
532 533 534 535 536 537 538
		mutex_unlock(&bdev->bd_fsfreeze_mutex);
		return sb;
	}

	sb = get_active_super(bdev);
	if (!sb)
		goto out;
539 540 541 542
	if (sb->s_op->freeze_super)
		error = sb->s_op->freeze_super(sb);
	else
		error = freeze_super(sb);
543 544 545
	if (error) {
		deactivate_super(sb);
		bdev->bd_fsfreeze_count--;
546
		mutex_unlock(&bdev->bd_fsfreeze_mutex);
547
		return ERR_PTR(error);
548
	}
549
	deactivate_super(sb);
550
 out:
551 552
	sync_blockdev(bdev);
	mutex_unlock(&bdev->bd_fsfreeze_mutex);
553
	return sb;	/* thaw_bdev releases s->s_umount */
554 555 556 557 558 559 560 561 562 563 564 565
}
EXPORT_SYMBOL(freeze_bdev);

/**
 * thaw_bdev  -- unlock filesystem
 * @bdev:	blockdevice to unlock
 * @sb:		associated superblock
 *
 * Unlocks the filesystem and marks it writeable again after freeze_bdev().
 */
int thaw_bdev(struct block_device *bdev, struct super_block *sb)
{
566
	int error = -EINVAL;
567 568

	mutex_lock(&bdev->bd_fsfreeze_mutex);
569
	if (!bdev->bd_fsfreeze_count)
570
		goto out;
571 572 573

	error = 0;
	if (--bdev->bd_fsfreeze_count > 0)
574
		goto out;
575 576

	if (!sb)
577
		goto out;
578

579 580 581 582
	if (sb->s_op->thaw_super)
		error = sb->s_op->thaw_super(sb);
	else
		error = thaw_super(sb);
583
	if (error)
584 585
		bdev->bd_fsfreeze_count++;
out:
586
	mutex_unlock(&bdev->bd_fsfreeze_mutex);
587
	return error;
588 589 590
}
EXPORT_SYMBOL(thaw_bdev);

Linus Torvalds's avatar
Linus Torvalds committed
591 592 593 594 595 596 597 598 599 600
static int blkdev_writepage(struct page *page, struct writeback_control *wbc)
{
	return block_write_full_page(page, blkdev_get_block, wbc);
}

static int blkdev_readpage(struct file * file, struct page * page)
{
	return block_read_full_page(page, blkdev_get_block);
}

601 602 603 604 605 606
static int blkdev_readpages(struct file *file, struct address_space *mapping,
			struct list_head *pages, unsigned nr_pages)
{
	return mpage_readpages(mapping, pages, nr_pages, blkdev_get_block);
}

Nick Piggin's avatar
Nick Piggin committed
607 608 609
static int blkdev_write_begin(struct file *file, struct address_space *mapping,
			loff_t pos, unsigned len, unsigned flags,
			struct page **pagep, void **fsdata)
Linus Torvalds's avatar
Linus Torvalds committed
610
{
611 612
	return block_write_begin(mapping, pos, len, flags, pagep,
				 blkdev_get_block);
Linus Torvalds's avatar
Linus Torvalds committed
613 614
}

Nick Piggin's avatar
Nick Piggin committed
615 616 617
static int blkdev_write_end(struct file *file, struct address_space *mapping,
			loff_t pos, unsigned len, unsigned copied,
			struct page *page, void *fsdata)
Linus Torvalds's avatar
Linus Torvalds committed
618
{
Nick Piggin's avatar
Nick Piggin committed
619 620 621 622
	int ret;
	ret = block_write_end(file, mapping, pos, len, copied, page, fsdata);

	unlock_page(page);
623
	put_page(page);
Nick Piggin's avatar
Nick Piggin committed
624 625

	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
626 627 628 629
}

/*
 * private llseek:
Al Viro's avatar
Al Viro committed
630
 * for a block special file file_inode(file)->i_size is zero
Linus Torvalds's avatar
Linus Torvalds committed
631 632
 * so we compute the size by hand (just as in block_read/write above)
 */
633
static loff_t block_llseek(struct file *file, loff_t offset, int whence)
Linus Torvalds's avatar
Linus Torvalds committed
634
{
635
	struct inode *bd_inode = bdev_file_inode(file);
Linus Torvalds's avatar
Linus Torvalds committed
636 637
	loff_t retval;

Al Viro's avatar
Al Viro committed
638
	inode_lock(bd_inode);
639
	retval = fixed_size_llseek(file, offset, whence, i_size_read(bd_inode));
Al Viro's avatar
Al Viro committed
640
	inode_unlock(bd_inode);
Linus Torvalds's avatar
Linus Torvalds committed
641 642 643
	return retval;
}
	
644
int blkdev_fsync(struct file *filp, loff_t start, loff_t end, int datasync)
Linus Torvalds's avatar
Linus Torvalds committed
645
{
646
	struct inode *bd_inode = bdev_file_inode(filp);
647
	struct block_device *bdev = I_BDEV(bd_inode);
648
	int error;
649
	
650
	error = file_write_and_wait_range(filp, start, end);
651 652
	if (error)
		return error;
653

654 655 656 657 658
	/*
	 * There is no need to serialise calls to blkdev_issue_flush with
	 * i_mutex and doing so causes performance issues with concurrent
	 * O_SYNC writers to a block device.
	 */
659
	error = blkdev_issue_flush(bdev, GFP_KERNEL, NULL);
660 661
	if (error == -EOPNOTSUPP)
		error = 0;
662

663
	return error;
Linus Torvalds's avatar
Linus Torvalds committed
664
}
665
EXPORT_SYMBOL(blkdev_fsync);
Linus Torvalds's avatar
Linus Torvalds committed
666

667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686
/**
 * bdev_read_page() - Start reading a page from a block device
 * @bdev: The device to read the page from
 * @sector: The offset on the device to read the page to (need not be aligned)
 * @page: The page to read
 *
 * On entry, the page should be locked.  It will be unlocked when the page
 * has been read.  If the block driver implements rw_page synchronously,
 * that will be true on exit from this function, but it need not be.
 *
 * Errors returned by this function are usually "soft", eg out of memory, or
 * queue full; callers should try a different route to read this page rather
 * than propagate an error back up the stack.
 *
 * Return: negative errno if an error occurs, 0 if submission was successful.
 */
int bdev_read_page(struct block_device *bdev, sector_t sector,
			struct page *page)
{
	const struct block_device_operations *ops = bdev->bd_disk->fops;
687 688
	int result = -EOPNOTSUPP;

689
	if (!ops->rw_page || bdev_get_integrity(bdev))
690 691
		return result;

692
	result = blk_queue_enter(bdev->bd_queue, false);
693 694
	if (result)
		return result;
695
	result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, false);
696 697
	blk_queue_exit(bdev->bd_queue);
	return result;
698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724
}
EXPORT_SYMBOL_GPL(bdev_read_page);

/**
 * bdev_write_page() - Start writing a page to a block device
 * @bdev: The device to write the page to
 * @sector: The offset on the device to write the page to (need not be aligned)
 * @page: The page to write
 * @wbc: The writeback_control for the write
 *
 * On entry, the page should be locked and not currently under writeback.
 * On exit, if the write started successfully, the page will be unlocked and
 * under writeback.  If the write failed already (eg the driver failed to
 * queue the page to the device), the page will still be locked.  If the
 * caller is a ->writepage implementation, it will need to unlock the page.
 *
 * Errors returned by this function are usually "soft", eg out of memory, or
 * queue full; callers should try a different route to write this page rather
 * than propagate an error back up the stack.
 *
 * Return: negative errno if an error occurs, 0 if submission was successful.
 */
int bdev_write_page(struct block_device *bdev, sector_t sector,
			struct page *page, struct writeback_control *wbc)
{
	int result;
	const struct block_device_operations *ops = bdev->bd_disk->fops;
725

726
	if (!ops->rw_page || bdev_get_integrity(bdev))
727
		return -EOPNOTSUPP;
728
	result = blk_queue_enter(bdev->bd_queue, false);
729 730 731
	if (result)
		return result;

732
	set_page_writeback(page);
733
	result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, true);
734
	if (result) {
735
		end_page_writeback(page);
736 737
	} else {
		clean_page_buffers(page);
738
		unlock_page(page);
739
	}
740
	blk_queue_exit(bdev->bd_queue);
741 742 743 744
	return result;
}
EXPORT_SYMBOL_GPL(bdev_write_page);

Linus Torvalds's avatar
Linus Torvalds committed
745 746 747 748 749
/*
 * pseudo-fs
 */

static  __cacheline_aligned_in_smp DEFINE_SPINLOCK(bdev_lock);
750
static struct kmem_cache * bdev_cachep __read_mostly;
Linus Torvalds's avatar
Linus Torvalds committed
751 752 753

static struct inode *bdev_alloc_inode(struct super_block *sb)
{
754
	struct bdev_inode *ei = kmem_cache_alloc(bdev_cachep, GFP_KERNEL);
Linus Torvalds's avatar
Linus Torvalds committed
755 756 757 758 759
	if (!ei)
		return NULL;
	return &ei->vfs_inode;
}

Nick Piggin's avatar
Nick Piggin committed
760
static void bdev_i_callback(struct rcu_head *head)
Linus Torvalds's avatar
Linus Torvalds committed
761
{
Nick Piggin's avatar
Nick Piggin committed
762
	struct inode *inode = container_of(head, struct inode, i_rcu);
Linus Torvalds's avatar
Linus Torvalds committed
763 764 765 766 767
	struct bdev_inode *bdi = BDEV_I(inode);

	kmem_cache_free(bdev_cachep, bdi);
}

Nick Piggin's avatar
Nick Piggin committed
768 769 770 771 772
static void bdev_destroy_inode(struct inode *inode)
{
	call_rcu(&inode->i_rcu, bdev_i_callback);
}

773
static void init_once(void *foo)
Linus Torvalds's avatar
Linus Torvalds committed
774 775 776 777
{
	struct bdev_inode *ei = (struct bdev_inode *) foo;
	struct block_device *bdev = &ei->bdev;

778 779 780
	memset(bdev, 0, sizeof(*bdev));
	mutex_init(&bdev->bd_mutex);
	INIT_LIST_HEAD(&bdev->bd_list);
781 782 783
#ifdef CONFIG_SYSFS
	INIT_LIST_HEAD(&bdev->bd_holder_disks);
#endif
784
	bdev->bd_bdi = &noop_backing_dev_info;
785
	inode_init_once(&ei->vfs_inode);
786 787
	/* Initialize mutex for freeze. */
	mutex_init(&bdev->bd_fsfreeze_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
788 789
}

790
static void bdev_evict_inode(struct inode *inode)
Linus Torvalds's avatar
Linus Torvalds committed
791 792
{
	struct block_device *bdev = &BDEV_I(inode)->bdev;
793
	truncate_inode_pages_final(&inode->i_data);
794
	invalidate_inode_buffers(inode); /* is it needed here? */
795
	clear_inode(inode);
Linus Torvalds's avatar
Linus Torvalds committed
796 797 798
	spin_lock(&bdev_lock);
	list_del_init(&bdev->bd_list);
	spin_unlock(&bdev_lock);
799 800
	/* Detach inode from wb early as bdi_put() may free bdi->wb */
	inode_detach_wb(inode);
801
	if (bdev->bd_bdi != &noop_backing_dev_info) {
802
		bdi_put(bdev->bd_bdi);
803 804
		bdev->bd_bdi = &noop_backing_dev_info;
	}
Linus Torvalds's avatar
Linus Torvalds committed
805 806
}

807
static const struct super_operations bdev_sops = {
Linus Torvalds's avatar
Linus Torvalds committed
808 809 810 811
	.statfs = simple_statfs,
	.alloc_inode = bdev_alloc_inode,
	.destroy_inode = bdev_destroy_inode,
	.drop_inode = generic_delete_inode,
812
	.evict_inode = bdev_evict_inode,
Linus Torvalds's avatar
Linus Torvalds committed
813 814
};

Al Viro's avatar
Al Viro committed
815 816
static struct dentry *bd_mount(struct file_system_type *fs_type,
	int flags, const char *dev_name, void *data)
Linus Torvalds's avatar
Linus Torvalds committed
817
{
818 819
	struct dentry *dent;
	dent = mount_pseudo(fs_type, "bdev:", &bdev_sops, NULL, BDEVFS_MAGIC);
820
	if (!IS_ERR(dent))
821 822
		dent->d_sb->s_iflags |= SB_I_CGROUPWB;
	return dent;
Linus Torvalds's avatar
Linus Torvalds committed
823 824 825 826
}

static struct file_system_type bd_type = {
	.name		= "bdev",
Al Viro's avatar
Al Viro committed
827
	.mount		= bd_mount,
Linus Torvalds's avatar
Linus Torvalds committed
828 829 830
	.kill_sb	= kill_anon_super,
};

Tejun Heo's avatar
Tejun Heo committed
831 832
struct super_block *blockdev_superblock __read_mostly;
EXPORT_SYMBOL_GPL(blockdev_superblock);
Linus Torvalds's avatar
Linus Torvalds committed
833 834 835 836

void __init bdev_cache_init(void)
{
	int err;
837
	static struct vfsmount *bd_mnt;
838

Linus Torvalds's avatar
Linus Torvalds committed
839
	bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode),
840
			0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
841
				SLAB_MEM_SPREAD|SLAB_ACCOUNT|SLAB_PANIC),
842
			init_once);
Linus Torvalds's avatar
Linus Torvalds committed
843 844 845 846 847 848
	err = register_filesystem(&bd_type);
	if (err)
		panic("Cannot register bdev pseudo-fs");
	bd_mnt = kern_mount(&bd_type);
	if (IS_ERR(bd_mnt))
		panic("Cannot create bdev pseudo-fs");
849
	blockdev_superblock = bd_mnt->mnt_sb;   /* For writeback */
Linus Torvalds's avatar
Linus Torvalds committed
850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874
}

/*
 * Most likely _very_ bad one - but then it's hardly critical for small
 * /dev and can be fixed when somebody will need really large one.
 * Keep in mind that it will be fed through icache hash function too.
 */
static inline unsigned long hash(dev_t dev)
{
	return MAJOR(dev)+MINOR(dev);
}

static int bdev_test(struct inode *inode, void *data)
{
	return BDEV_I(inode)->bdev.bd_dev == *(dev_t *)data;
}

static int bdev_set(struct inode *inode, void *data)
{
	BDEV_I(inode)->bdev.bd_dev = *(dev_t *)data;
	return 0;
}

static LIST_HEAD(all_bdevs);

875 876 877 878 879 880 881 882 883 884 885 886 887 888 889
/*
 * If there is a bdev inode for this device, unhash it so that it gets evicted
 * as soon as last inode reference is dropped.
 */
void bdev_unhash_inode(dev_t dev)
{
	struct inode *inode;

	inode = ilookup5(blockdev_superblock, hash(dev), bdev_test, &dev);
	if (inode) {
		remove_inode_hash(inode);
		iput(inode);
	}
}

Linus Torvalds's avatar
Linus Torvalds committed
890 891 892 893 894
struct block_device *bdget(dev_t dev)
{
	struct block_device *bdev;
	struct inode *inode;

895
	inode = iget5_locked(blockdev_superblock, hash(dev),
Linus Torvalds's avatar
Linus Torvalds committed
896 897 898 899 900 901 902 903 904
			bdev_test, bdev_set, &dev);

	if (!inode)
		return NULL;

	bdev = &BDEV_I(inode)->bdev;

	if (inode->i_state & I_NEW) {
		bdev->bd_contains = NULL;
905
		bdev->bd_super = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
906
		bdev->bd_inode = inode;
Fabian Frederick's avatar
Fabian Frederick committed
907
		bdev->bd_block_size = i_blocksize(inode);
Linus Torvalds's avatar
Linus Torvalds committed
908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924
		bdev->bd_part_count = 0;
		bdev->bd_invalidated = 0;
		inode->i_mode = S_IFBLK;
		inode->i_rdev = dev;
		inode->i_bdev = bdev;
		inode->i_data.a_ops = &def_blk_aops;
		mapping_set_gfp_mask(&inode->i_data, GFP_USER);
		spin_lock(&bdev_lock);
		list_add(&bdev->bd_list, &all_bdevs);
		spin_unlock(&bdev_lock);
		unlock_new_inode(inode);
	}
	return bdev;
}

EXPORT_SYMBOL(bdget);

925 926 927 928 929 930
/**
 * bdgrab -- Grab a reference to an already referenced block device
 * @bdev:	Block device to grab a reference to.
 */
struct block_device *bdgrab(struct block_device *bdev)
{
Al Viro's avatar
Al Viro committed
931
	ihold(bdev->bd_inode);
932 933
	return bdev;
}
934
EXPORT_SYMBOL(bdgrab);
935

Linus Torvalds's avatar
Linus Torvalds committed
936 937
long nr_blockdev_pages(void)
{
938
	struct block_device *bdev;
Linus Torvalds's avatar
Linus Torvalds committed
939 940
	long ret = 0;
	spin_lock(&bdev_lock);
941
	list_for_each_entry(bdev, &all_bdevs, bd_list) {
Linus Torvalds's avatar
Linus Torvalds committed
942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957
		ret += bdev->bd_inode->i_mapping->nrpages;
	}
	spin_unlock(&bdev_lock);
	return ret;
}

void bdput(struct block_device *bdev)
{
	iput(bdev->bd_inode);
}

EXPORT_SYMBOL(bdput);
 
static struct block_device *bd_acquire(struct inode *inode)
{
	struct block_device *bdev;
958

Linus Torvalds's avatar
Linus Torvalds committed
959 960
	spin_lock(&bdev_lock);
	bdev = inode->i_bdev;
961
	if (bdev && !inode_unhashed(bdev->bd_inode)) {
962
		bdgrab(bdev);
Linus Torvalds's avatar
Linus Torvalds committed
963 964 965 966
		spin_unlock(&bdev_lock);
		return bdev;
	}
	spin_unlock(&bdev_lock);
967

968 969 970 971 972 973 974 975 976
	/*
	 * i_bdev references block device inode that was already shut down
	 * (corresponding device got removed).  Remove the reference and look
	 * up block device inode again just in case new device got
	 * reestablished under the same device number.
	 */
	if (bdev)
		bd_forget(inode);

Linus Torvalds's avatar
Linus Torvalds committed
977 978 979
	bdev = bdget(inode->i_rdev);
	if (bdev) {
		spin_lock(&bdev_lock);
980 981
		if (!inode->i_bdev) {
			/*
Al Viro's avatar
Al Viro committed
982
			 * We take an additional reference to bd_inode,
983 984 985 986
			 * and it's released in clear_inode() of inode.
			 * So, we can access it via ->i_mapping always
			 * without igrab().
			 */
987
			bdgrab(bdev);
988 989 990
			inode->i_bdev = bdev;
			inode->i_mapping = bdev->bd_inode->i_mapping;
		}
Linus Torvalds's avatar
Linus Torvalds committed
991 992 993 994 995 996 997 998 999
		spin_unlock(&bdev_lock);
	}
	return bdev;
}

/* Call when you free inode */

void bd_forget(struct inode *inode)
{
1000 1001
	struct block_device *bdev = NULL;

Linus Torvalds's avatar
Linus Torvalds committed
1002
	spin_lock(&bdev_lock);
1003 1004
	if (!sb_is_blkdev_sb(inode->i_sb))
		bdev = inode->i_bdev;
Al Viro's avatar
Al Viro committed
1005 1006
	inode->i_bdev = NULL;
	inode->i_mapping = &inode->i_data;
Linus Torvalds's avatar
Linus Torvalds committed
1007
	spin_unlock(&bdev_lock);
1008 1009

	if (bdev)
1010
		bdput(bdev);
Linus Torvalds's avatar
Linus Torvalds committed
1011 1012
}

Tejun Heo's avatar
Tejun Heo committed
1013 1014 1015 1016 1017 1018
/**
 * bd_may_claim - test whether a block device can be claimed
 * @bdev: block device of interest
 * @whole: whole block device containing @bdev, may equal @bdev
 * @holder: holder trying to claim @bdev
 *
Lucas De Marchi's avatar
Lucas De Marchi committed
1019
 * Test whether @bdev can be claimed by @holder.
Tejun Heo's avatar
Tejun Heo committed
1020 1021 1022 1023 1024 1025 1026 1027 1028
 *
 * CONTEXT:
 * spin_lock(&bdev_lock).
 *
 * RETURNS:
 * %true if @bdev can be claimed, %false otherwise.
 */
static bool bd_may_claim(struct block_device *bdev, struct block_device *whole,
			 void *holder)
Linus Torvalds's avatar
Linus Torvalds committed
1029 1030
{
	if (bdev->bd_holder == holder)
Tejun Heo's avatar
Tejun Heo committed
1031
		return true;	 /* already a holder */
Linus Torvalds's avatar
Linus Torvalds committed
1032
	else if (bdev->bd_holder != NULL)
Tejun Heo's avatar
Tejun Heo committed
1033
		return false; 	 /* held by someone else */
1034
	else if (whole == bdev)
Tejun Heo's avatar
Tejun Heo committed
1035
		return true;  	 /* is a whole device which isn't held */
Linus Torvalds's avatar
Linus Torvalds committed
1036

1037
	else if (whole->bd_holder == bd_may_claim)
Tejun Heo's avatar
Tejun Heo committed
1038 1039 1040
		return true; 	 /* is a partition of a device that is being partitioned */
	else if (whole->bd_holder != NULL)
		return false;	 /* is a partition of a held device */
Linus Torvalds's avatar
Linus Torvalds committed
1041
	else
Tejun Heo's avatar
Tejun Heo committed
1042 1043 1044
		return true;	 /* is a partition of an un-held device */
}

1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
/**
 * bd_prepare_to_claim - prepare to claim a block device
 * @bdev: block device of interest
 * @whole: the whole device containing @bdev, may equal @bdev
 * @holder: holder trying to claim @bdev
 *
 * Prepare to claim @bdev.  This function fails if @bdev is already
 * claimed by another holder and waits if another claiming is in
 * progress.  This function doesn't actually claim.  On successful
 * return, the caller has ownership of bd_claiming and bd_holder[s].
 *
 * CONTEXT:
 * spin_lock(&bdev_lock).  Might release bdev_lock, sleep and regrab
 * it multiple times.
 *
 * RETURNS:
 * 0 if @bdev can be claimed, -EBUSY otherwise.
 */
static int bd_prepare_to_claim(struct block_device *bdev,
			       struct block_device *whole, void *holder)
{
retry:
	/* if someone else claimed, fail */
	if (!bd_may_claim(bdev, whole, holder))
		return -EBUSY;

1071 1072
	/* if claiming is already in progress, wait for it to finish */
	if (whole->bd_claiming) {
1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095
		wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0);
		DEFINE_WAIT(wait);

		prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
		spin_unlock(&bdev_lock);
		schedule();
		finish_wait(wq, &wait);
		spin_lock(&bdev_lock);
		goto retry;
	}

	/* yay, all mine */
	return 0;
}

/**
 * bd_start_claiming - start claiming a block device
 * @bdev: block device of interest
 * @holder: holder trying to claim @bdev
 *
 * @bdev is about to be opened exclusively.  Check @bdev can be opened
 * exclusively and mark that an exclusive open is in progress.  Each
 * successful call to this function must be matched with a call to
Nick Piggin's avatar
Nick Piggin committed
1096 1097 1098 1099 1100 1101 1102
 * either bd_finish_claiming() or bd_abort_claiming() (which do not
 * fail).
 *
 * This function is used to gain exclusive access to the block device
 * without actually causing other exclusive open attempts to fail. It
 * should be used when the open sequence itself requires exclusive
 * access but may subsequently fail.
1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127
 *
 * CONTEXT:
 * Might sleep.
 *
 * RETURNS:
 * Pointer to the block device containing @bdev on success, ERR_PTR()
 * value on failure.
 */
static struct block_device *bd_start_claiming(struct block_device *bdev,
					      void *holder)
{
	struct gendisk *disk;
	struct block_device *whole;
	int partno, err;

	might_sleep();

	/*
	 * @bdev might not have been initialized properly yet, look up
	 * and grab the outer block device the hard way.
	 */
	disk = get_gendisk(bdev->bd_dev, &partno);
	if (!disk)
		return ERR_PTR(-ENXIO);

1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140
	/*
	 * Normally, @bdev should equal what's returned from bdget_disk()
	 * if partno is 0; however, some drivers (floppy) use multiple
	 * bdev's for the same physical device and @bdev may be one of the
	 * aliases.  Keep @bdev if partno is 0.  This means claimer
	 * tracking is broken for those devices but it has always been that
	 * way.
	 */
	if (partno)
		whole = bdget_disk(disk, 0);
	else
		whole = bdgrab(bdev);

1141
	module_put(disk->fops->owner);
1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160
	put_disk(disk);
	if (!whole)
		return ERR_PTR(-ENOMEM);

	/* prepare to claim, if successful, mark claiming in progress */
	spin_lock(&bdev_lock);

	err = bd_prepare_to_claim(bdev, whole, holder);
	if (err == 0) {
		whole->bd_claiming = holder;
		spin_unlock(&bdev_lock);
		return whole;
	} else {
		spin_unlock(&bdev_lock);
		bdput(whole);
		return ERR_PTR(err);
	}
}

1161
#ifdef CONFIG_SYSFS
1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178
struct bd_holder_disk {
	struct list_head	list;
	struct gendisk		*disk;
	int			refcnt;
};

static struct bd_holder_disk *bd_find_holder_disk(struct block_device *bdev,
						  struct gendisk *disk)
{
	struct bd_holder_disk *holder;

	list_for_each_entry(holder, &bdev->bd_holder_disks, list)
		if (holder->disk == disk)
			return holder;
	return NULL;
}

1179
static int add_symlink(struct kobject *from, struct kobject *to)
1180
{
1181
	return sysfs_create_link(from, to, kobject_name(to));
1182 1183 1184 1185 1186 1187 1188
}

static void del_symlink(struct kobject *from, struct kobject *to)
{
	sysfs_remove_link(from, kobject_name(to));
}

1189
/**
1190 1191 1192
 * bd_link_disk_holder - create symlinks between holding disk and slave bdev
 * @bdev: the claimed slave bdev
 * @disk: the holding disk
1193
 *
1194 1195
 * DON'T USE THIS UNLESS YOU'RE ALREADY USING IT.
 *
1196
 * This functions creates the following sysfs symlinks.
1197
 *
1198 1199
 * - from "slaves" directory of the holder @disk to the claimed @bdev
 * - from "holders" directory of the @bdev to the holder @disk
1200
 *
1201 1202
 * For example, if /dev/dm-0 maps to /dev/sda and disk for dm-0 is
 * passed to bd_link_disk_holder(), then:
1203
 *
1204 1205
 *   /sys/block/dm-0/slaves/sda --> /sys/block/sda
 *   /sys/block/sda/holders/dm-0 --> /sys/block/dm-0
1206
 *
1207 1208 1209
 * The caller must have claimed @bdev before calling this function and
 * ensure that both @bdev and @disk are valid during the creation and
 * lifetime of these symlinks.
1210
 *
1211 1212
 * CONTEXT:
 * Might sleep.
1213
 *
1214 1215
 * RETURNS:
 * 0 on success, -errno on failure.
1216
 */
1217
int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk)
1218
{
1219
	struct bd_holder_disk *holder;
1220
	int ret = 0;
1221

1222
	mutex_lock(&bdev->bd_mutex);
1223

1224
	WARN_ON_ONCE(!bdev->bd_holder);
1225

1226 1227 1228
	/* FIXME: remove the following once add_disk() handles errors */
	if (WARN_ON(!disk->slave_dir || !bdev->bd_part->holder_dir))
		goto out_unlock;
1229

1230 1231 1232
	holder = bd_find_holder_disk(bdev, disk);
	if (holder) {
		holder->refcnt++;
1233
		goto out_unlock;
1234
	}
1235

1236 1237 1238
	holder = kzalloc(sizeof(*holder), GFP_KERNEL);
	if (!holder) {
		ret = -ENOMEM;
1239 1240
		goto out_unlock;
	}
1241

1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252
	INIT_LIST_HEAD(&holder->list);
	holder->disk = disk;
	holder->refcnt = 1;

	ret = add_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
	if (ret)
		goto out_free;

	ret = add_symlink(bdev->bd_part->holder_dir, &disk_to_dev(disk)->kobj);
	if (ret)
		goto out_del;
1253 1254 1255 1256 1257
	/*
	 * bdev could be deleted beneath us which would implicitly destroy
	 * the holder directory.  Hold on to it.
	 */
	kobject_get(bdev->bd_part->holder_dir);
1258 1259 1260 1261 1262 1263 1264 1265

	list_add(&holder->list, &bdev->bd_holder_disks);
	goto out_unlock;

out_del:
	del_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
out_free:
	kfree(holder);
1266
out_unlock:
1267
	mutex_unlock(&bdev->bd_mutex);
1268
	return ret;
1269
}
1270
EXPORT_SYMBOL_GPL(bd_link_disk_holder);
1271

1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282
/**
 * bd_unlink_disk_holder - destroy symlinks created by bd_link_disk_holder()
 * @bdev: the calimed slave bdev
 * @disk: the holding disk
 *
 * DON'T USE THIS UNLESS YOU'RE ALREADY USING IT.
 *
 * CONTEXT:
 * Might sleep.
 */
void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk)
1283
{
1284
	struct bd_holder_disk *holder;
1285

1286
	mutex_lock(&bdev->bd_mutex);
1287

1288 1289 1290 1291 1292 1293
	holder = bd_find_holder_disk(bdev, disk);

	if (!WARN_ON_ONCE(holder == NULL) && !--holder->refcnt) {
		del_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
		del_symlink(bdev->bd_part->holder_dir,
			    &disk_to_dev(disk)->kobj);
1294
		kobject_put(bdev->bd_part->holder_dir);
1295 1296 1297 1298 1299
		list_del_init(&holder->list);
		kfree(holder);
	}

	mutex_unlock(&bdev->bd_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
1300
}
1301
EXPORT_SYMBOL_GPL(bd_unlink_disk_holder);
1302
#endif
Linus Torvalds's avatar
Linus Torvalds committed
1303

1304 1305 1306 1307
/**
 * flush_disk - invalidates all buffer-cache entries on a disk
 *
 * @bdev:      struct block device to be flushed
1308
 * @kill_dirty: flag to guide handling of dirty inodes
1309 1310 1311 1312 1313
 *
 * Invalidates all buffer-cache entries on a disk. It should be called
 * when a disk has been changed -- either by a media change or online
 * resize.
 */
1314
static void flush_disk(struct block_device *bdev, bool kill_dirty)
1315
{
1316
	if (__invalidate_device(bdev, kill_dirty)) {
1317
		printk(KERN_WARNING "VFS: busy inodes on changed media or "
1318 1319
		       "resized disk %s\n",
		       bdev->bd_disk ? bdev->bd_disk->disk_name : "");
1320 1321 1322 1323
	}

	if (!bdev->bd_disk)
		return;
Tejun Heo's avatar
Tejun Heo committed
1324
	if (disk_part_scan_enabled(bdev->bd_disk))
1325 1326 1327
		bdev->bd_invalidated = 1;
}

1328
/**
1329
 * check_disk_size_change - checks for disk size change and adjusts bdev size.
1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344
 * @disk: struct gendisk to check
 * @bdev: struct bdev to adjust.
 *
 * This routine checks to see if the bdev size does not match the disk size
 * and adjusts it if it differs.
 */
void check_disk_size_change(struct gendisk *disk, struct block_device *bdev)
{
	loff_t disk_size, bdev_size;

	disk_size = (loff_t)get_capacity(disk) << 9;
	bdev_size = i_size_read(bdev->bd_inode);
	if (disk_size != bdev_size) {
		printk(KERN_INFO
		       "%s: detected capacity change from %lld to %lld\n",
1345
		       disk->disk_name, bdev_size, disk_size);
1346
		i_size_write(bdev->bd_inode, disk_size);
1347
		flush_disk(bdev, false);
1348 1349 1350 1351
	}
}
EXPORT_SYMBOL(check_disk_size_change);

1352
/**
1353
 * revalidate_disk - wrapper for lower-level driver's revalidate_disk call-back
1354 1355 1356 1357 1358 1359 1360 1361
 * @disk: struct gendisk to be revalidated
 *
 * This routine is a wrapper for lower-level driver's revalidate_disk
 * call-backs.  It is used to do common pre and post operations needed
 * for all revalidate_disk operations.
 */
int revalidate_disk(struct gendisk *disk)