loop.c 50.7 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
/*
 *  linux/drivers/block/loop.c
 *
 *  Written by Theodore Ts'o, 3/29/93
 *
 * Copyright 1993 by Theodore Ts'o.  Redistribution of this file is
 * permitted under the GNU General Public License.
 *
 * DES encryption plus some minor changes by Werner Almesberger, 30-MAY-1993
 * more DES encryption plus IDEA encryption by Nicholas J. Leon, June 20, 1996
 *
 * Modularized and updated for 1.1.16 kernel - Mitch Dsouza 28th May 1994
 * Adapted for 1.3.59 kernel - Andries Brouwer, 1 Feb 1996
 *
 * Fixed do_loop_request() re-entrancy - Vincent.Renardias@waw.com Mar 20, 1997
 *
 * Added devfs support - Richard Gooch <rgooch@atnf.csiro.au> 16-Jan-1998
 *
 * Handle sparse backing files correctly - Kenn Humborg, Jun 28, 1998
 *
 * Loadable modules and other fixes by AK, 1998
 *
 * Make real block number available to downstream transfer functions, enables
 * CBC (and relatives) mode encryption requiring unique IVs per data block.
 * Reed H. Petty, rhp@draper.net
 *
 * Maximum number of loop devices now dynamic via max_loop module parameter.
 * Russell Kroll <rkroll@exploits.org> 19990701
 *
 * Maximum number of loop devices when compiled-in now selectable by passing
 * max_loop=<1-255> to the kernel on boot.
32
 * Erik I. Bolsø, <eriki@himolde.no>, Oct 31, 1999
Linus Torvalds's avatar
Linus Torvalds committed
33 34 35 36 37 38 39 40 41 42
 *
 * Completely rewrite request handling to be make_request_fn style and
 * non blocking, pushing work to a helper thread. Lots of fixes from
 * Al Viro too.
 * Jens Axboe <axboe@suse.de>, Nov 2000
 *
 * Support up to 256 loop devices
 * Heinz Mauelshagen <mge@sistina.com>, Feb 2002
 *
 * Support for falling back on the write file operation when the address space
43
 * operations write_begin is not available on the backing filesystem.
Linus Torvalds's avatar
Linus Torvalds committed
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
 * Anton Altaparmakov, 16 Feb 2005
 *
 * Still To Fix:
 * - Advisory locking is ignored here.
 * - Should use an own CAP_* category instead of CAP_SYS_ADMIN
 *
 */

#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/stat.h>
#include <linux/errno.h>
#include <linux/major.h>
#include <linux/wait.h>
#include <linux/blkdev.h>
#include <linux/blkpg.h>
#include <linux/init.h>
#include <linux/swap.h>
#include <linux/slab.h>
66
#include <linux/compat.h>
Linus Torvalds's avatar
Linus Torvalds committed
67
#include <linux/suspend.h>
68
#include <linux/freezer.h>
69
#include <linux/mutex.h>
Linus Torvalds's avatar
Linus Torvalds committed
70 71 72
#include <linux/writeback.h>
#include <linux/completion.h>
#include <linux/highmem.h>
73
#include <linux/kthread.h>
74
#include <linux/splice.h>
75
#include <linux/sysfs.h>
76
#include <linux/miscdevice.h>
77
#include <linux/falloc.h>
Al Viro's avatar
Al Viro committed
78
#include <linux/uio.h>
Al Viro's avatar
Al Viro committed
79
#include "loop.h"
Linus Torvalds's avatar
Linus Torvalds committed
80 81 82

#include <asm/uaccess.h>

83
static DEFINE_IDR(loop_index_idr);
84
static DEFINE_MUTEX(loop_index_mutex);
85
static DEFINE_MUTEX(loop_ctl_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
86

87 88 89
static int max_part;
static int part_shift;

Linus Torvalds's avatar
Linus Torvalds committed
90 91 92 93 94
static int transfer_xor(struct loop_device *lo, int cmd,
			struct page *raw_page, unsigned raw_off,
			struct page *loop_page, unsigned loop_off,
			int size, sector_t real_block)
{
95 96
	char *raw_buf = kmap_atomic(raw_page) + raw_off;
	char *loop_buf = kmap_atomic(loop_page) + loop_off;
Linus Torvalds's avatar
Linus Torvalds committed
97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112
	char *in, *out, *key;
	int i, keysize;

	if (cmd == READ) {
		in = raw_buf;
		out = loop_buf;
	} else {
		in = loop_buf;
		out = raw_buf;
	}

	key = lo->lo_encrypt_key;
	keysize = lo->lo_encrypt_key_size;
	for (i = 0; i < size; i++)
		*out++ = *in++ ^ key[(i & 511) % keysize];

113 114
	kunmap_atomic(loop_buf);
	kunmap_atomic(raw_buf);
Linus Torvalds's avatar
Linus Torvalds committed
115 116 117 118 119 120 121 122 123 124 125 126 127
	cond_resched();
	return 0;
}

static int xor_init(struct loop_device *lo, const struct loop_info64 *info)
{
	if (unlikely(info->lo_encrypt_key_size <= 0))
		return -EINVAL;
	return 0;
}

static struct loop_func_table none_funcs = {
	.number = LO_CRYPT_NONE,
128
}; 
Linus Torvalds's avatar
Linus Torvalds committed
129 130 131 132 133

static struct loop_func_table xor_funcs = {
	.number = LO_CRYPT_XOR,
	.transfer = transfer_xor,
	.init = xor_init
134
}; 
Linus Torvalds's avatar
Linus Torvalds committed
135 136 137 138 139 140 141

/* xfer_funcs[0] is special - its release function is never called */
static struct loop_func_table *xfer_funcs[MAX_LO_CRYPT] = {
	&none_funcs,
	&xor_funcs
};

142
static loff_t get_size(loff_t offset, loff_t sizelimit, struct file *file)
Linus Torvalds's avatar
Linus Torvalds committed
143
{
144
	loff_t loopsize;
Linus Torvalds's avatar
Linus Torvalds committed
145 146

	/* Compute loopsize in bytes */
147 148 149 150
	loopsize = i_size_read(file->f_mapping->host);
	if (offset > 0)
		loopsize -= offset;
	/* offset is beyond i_size, weird but possible */
151 152
	if (loopsize < 0)
		return 0;
Linus Torvalds's avatar
Linus Torvalds committed
153

154 155
	if (sizelimit > 0 && sizelimit < loopsize)
		loopsize = sizelimit;
Linus Torvalds's avatar
Linus Torvalds committed
156 157 158 159 160 161 162
	/*
	 * Unfortunately, if we want to do I/O on the device,
	 * the number of 512-byte sectors has to fit into a sector_t.
	 */
	return loopsize >> 9;
}

163 164 165 166 167
static loff_t get_loop_size(struct loop_device *lo, struct file *file)
{
	return get_size(lo->lo_offset, lo->lo_sizelimit, file);
}

168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
static void __loop_update_dio(struct loop_device *lo, bool dio)
{
	struct file *file = lo->lo_backing_file;
	struct address_space *mapping = file->f_mapping;
	struct inode *inode = mapping->host;
	unsigned short sb_bsize = 0;
	unsigned dio_align = 0;
	bool use_dio;

	if (inode->i_sb->s_bdev) {
		sb_bsize = bdev_logical_block_size(inode->i_sb->s_bdev);
		dio_align = sb_bsize - 1;
	}

	/*
	 * We support direct I/O only if lo_offset is aligned with the
	 * logical I/O size of backing device, and the logical block
	 * size of loop is bigger than the backing device's and the loop
	 * needn't transform transfer.
	 *
	 * TODO: the above condition may be loosed in the future, and
	 * direct I/O may be switched runtime at that time because most
	 * of requests in sane appplications should be PAGE_SIZE algined
	 */
	if (dio) {
		if (queue_logical_block_size(lo->lo_queue) >= sb_bsize &&
				!(lo->lo_offset & dio_align) &&
				mapping->a_ops->direct_IO &&
				!lo->transfer)
			use_dio = true;
		else
			use_dio = false;
	} else {
		use_dio = false;
	}

	if (lo->use_dio == use_dio)
		return;

	/* flush dirty pages before changing direct IO */
	vfs_fsync(file, 0);

	/*
	 * The flag of LO_FLAGS_DIRECT_IO is handled similarly with
	 * LO_FLAGS_READ_ONLY, both are set from kernel, and losetup
	 * will get updated by ioctl(LOOP_GET_STATUS)
	 */
	blk_mq_freeze_queue(lo->lo_queue);
	lo->use_dio = use_dio;
	if (use_dio)
		lo->lo_flags |= LO_FLAGS_DIRECT_IO;
	else
		lo->lo_flags &= ~LO_FLAGS_DIRECT_IO;
	blk_mq_unfreeze_queue(lo->lo_queue);
}

Linus Torvalds's avatar
Linus Torvalds committed
224
static int
225
figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit)
Linus Torvalds's avatar
Linus Torvalds committed
226
{
227
	loff_t size = get_size(offset, sizelimit, lo->lo_backing_file);
Linus Torvalds's avatar
Linus Torvalds committed
228
	sector_t x = (sector_t)size;
229
	struct block_device *bdev = lo->lo_device;
Linus Torvalds's avatar
Linus Torvalds committed
230 231 232

	if (unlikely((loff_t)x != size))
		return -EFBIG;
233 234 235 236
	if (lo->lo_offset != offset)
		lo->lo_offset = offset;
	if (lo->lo_sizelimit != sizelimit)
		lo->lo_sizelimit = sizelimit;
237
	set_capacity(lo->lo_disk, x);
238 239 240
	bd_set_size(bdev, (loff_t)get_capacity(bdev->bd_disk) << 9);
	/* let user-space know about the new size */
	kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
241
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
242 243 244 245 246 247 248 249
}

static inline int
lo_do_transfer(struct loop_device *lo, int cmd,
	       struct page *rpage, unsigned roffs,
	       struct page *lpage, unsigned loffs,
	       int size, sector_t rblock)
{
250 251 252 253
	int ret;

	ret = lo->transfer(lo, cmd, rpage, roffs, lpage, loffs, size, rblock);
	if (likely(!ret))
Linus Torvalds's avatar
Linus Torvalds committed
254 255
		return 0;

256 257 258 259
	printk_ratelimited(KERN_ERR
		"loop: Transfer error at byte offset %llu, length %i.\n",
		(unsigned long long)rblock << 9, size);
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
260 261
}

262
static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos)
Linus Torvalds's avatar
Linus Torvalds committed
263
{
264
	struct iov_iter i;
Linus Torvalds's avatar
Linus Torvalds committed
265
	ssize_t bw;
Al Viro's avatar
Al Viro committed
266

267
	iov_iter_bvec(&i, ITER_BVEC | WRITE, bvec, 1, bvec->bv_len);
Linus Torvalds's avatar
Linus Torvalds committed
268

269
	file_start_write(file);
270
	bw = vfs_iter_write(file, &i, ppos);
271
	file_end_write(file);
272 273

	if (likely(bw ==  bvec->bv_len))
Linus Torvalds's avatar
Linus Torvalds committed
274
		return 0;
275 276 277 278

	printk_ratelimited(KERN_ERR
		"loop: Write error at byte offset %llu, length %i.\n",
		(unsigned long long)*ppos, bvec->bv_len);
Linus Torvalds's avatar
Linus Torvalds committed
279 280 281 282 283
	if (bw >= 0)
		bw = -EIO;
	return bw;
}

284 285
static int lo_write_simple(struct loop_device *lo, struct request *rq,
		loff_t pos)
Linus Torvalds's avatar
Linus Torvalds committed
286
{
287 288 289 290 291 292 293 294 295 296 297 298
	struct bio_vec bvec;
	struct req_iterator iter;
	int ret = 0;

	rq_for_each_segment(bvec, rq, iter) {
		ret = lo_write_bvec(lo->lo_backing_file, &bvec, &pos);
		if (ret < 0)
			break;
		cond_resched();
	}

	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
299 300
}

301
/*
302 303 304
 * This is the slow, transforming version that needs to double buffer the
 * data as it cannot do the transformations in place without having direct
 * access to the destination pages of the backing file.
Linus Torvalds's avatar
Linus Torvalds committed
305
 */
306 307
static int lo_write_transfer(struct loop_device *lo, struct request *rq,
		loff_t pos)
Linus Torvalds's avatar
Linus Torvalds committed
308
{
309
	struct bio_vec bvec, b;
Ming Lei's avatar
Ming Lei committed
310
	struct req_iterator iter;
311
	struct page *page;
312
	int ret = 0;
Linus Torvalds's avatar
Linus Torvalds committed
313

314 315 316
	page = alloc_page(GFP_NOIO);
	if (unlikely(!page))
		return -ENOMEM;
317

Ming Lei's avatar
Ming Lei committed
318
	rq_for_each_segment(bvec, rq, iter) {
319 320 321 322 323 324 325 326 327
		ret = lo_do_transfer(lo, WRITE, page, 0, bvec.bv_page,
			bvec.bv_offset, bvec.bv_len, pos >> 9);
		if (unlikely(ret))
			break;

		b.bv_page = page;
		b.bv_offset = 0;
		b.bv_len = bvec.bv_len;
		ret = lo_write_bvec(lo->lo_backing_file, &b, &pos);
Linus Torvalds's avatar
Linus Torvalds committed
328 329 330
		if (ret < 0)
			break;
	}
331 332

	__free_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
333 334 335
	return ret;
}

336 337
static int lo_read_simple(struct loop_device *lo, struct request *rq,
		loff_t pos)
Linus Torvalds's avatar
Linus Torvalds committed
338
{
339 340 341 342
	struct bio_vec bvec;
	struct req_iterator iter;
	struct iov_iter i;
	ssize_t len;
Linus Torvalds's avatar
Linus Torvalds committed
343

344 345 346 347 348
	rq_for_each_segment(bvec, rq, iter) {
		iov_iter_bvec(&i, ITER_BVEC, &bvec, 1, bvec.bv_len);
		len = vfs_iter_read(lo->lo_backing_file, &i, &pos);
		if (len < 0)
			return len;
Linus Torvalds's avatar
Linus Torvalds committed
349

350
		flush_dcache_page(bvec.bv_page);
351

352 353
		if (len != bvec.bv_len) {
			struct bio *bio;
Linus Torvalds's avatar
Linus Torvalds committed
354

355 356 357 358 359 360 361 362
			__rq_for_each_bio(bio, rq)
				zero_fill_bio(bio);
			break;
		}
		cond_resched();
	}

	return 0;
363 364
}

365 366
static int lo_read_transfer(struct loop_device *lo, struct request *rq,
		loff_t pos)
Linus Torvalds's avatar
Linus Torvalds committed
367
{
368 369 370 371 372 373
	struct bio_vec bvec, b;
	struct req_iterator iter;
	struct iov_iter i;
	struct page *page;
	ssize_t len;
	int ret = 0;
Linus Torvalds's avatar
Linus Torvalds committed
374

375 376 377
	page = alloc_page(GFP_NOIO);
	if (unlikely(!page))
		return -ENOMEM;
378

379 380
	rq_for_each_segment(bvec, rq, iter) {
		loff_t offset = pos;
381

382 383 384
		b.bv_page = page;
		b.bv_offset = 0;
		b.bv_len = bvec.bv_len;
385

386 387 388 389 390 391
		iov_iter_bvec(&i, ITER_BVEC, &b, 1, b.bv_len);
		len = vfs_iter_read(lo->lo_backing_file, &i, &pos);
		if (len < 0) {
			ret = len;
			goto out_free_page;
		}
Linus Torvalds's avatar
Linus Torvalds committed
392

393 394 395 396
		ret = lo_do_transfer(lo, READ, page, 0, bvec.bv_page,
			bvec.bv_offset, len, offset >> 9);
		if (ret)
			goto out_free_page;
Linus Torvalds's avatar
Linus Torvalds committed
397

398
		flush_dcache_page(bvec.bv_page);
399

400
		if (len != bvec.bv_len) {
Ming Lei's avatar
Ming Lei committed
401 402 403 404
			struct bio *bio;

			__rq_for_each_bio(bio, rq)
				zero_fill_bio(bio);
Linus Torvalds's avatar
Linus Torvalds committed
405
			break;
406
		}
Linus Torvalds's avatar
Linus Torvalds committed
407
	}
408 409 410 411 412

	ret = 0;
out_free_page:
	__free_page(page);
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
413 414
}

415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448
static int lo_discard(struct loop_device *lo, struct request *rq, loff_t pos)
{
	/*
	 * We use punch hole to reclaim the free space used by the
	 * image a.k.a. discard. However we do not support discard if
	 * encryption is enabled, because it may give an attacker
	 * useful information.
	 */
	struct file *file = lo->lo_backing_file;
	int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
	int ret;

	if ((!file->f_op->fallocate) || lo->lo_encrypt_key_size) {
		ret = -EOPNOTSUPP;
		goto out;
	}

	ret = file->f_op->fallocate(file, mode, pos, blk_rq_bytes(rq));
	if (unlikely(ret && ret != -EINVAL && ret != -EOPNOTSUPP))
		ret = -EIO;
 out:
	return ret;
}

static int lo_req_flush(struct loop_device *lo, struct request *rq)
{
	struct file *file = lo->lo_backing_file;
	int ret = vfs_fsync(file, 0);
	if (unlikely(ret && ret != -EINVAL))
		ret = -EIO;

	return ret;
}

Ming Lei's avatar
Ming Lei committed
449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473
static inline void handle_partial_read(struct loop_cmd *cmd, long bytes)
{
	if (bytes < 0 || (cmd->rq->cmd_flags & REQ_WRITE))
		return;

	if (unlikely(bytes < blk_rq_bytes(cmd->rq))) {
		struct bio *bio = cmd->rq->bio;

		bio_advance(bio, bytes);
		zero_fill_bio(bio);
	}
}

static void lo_rw_aio_complete(struct kiocb *iocb, long ret, long ret2)
{
	struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb);
	struct request *rq = cmd->rq;

	handle_partial_read(cmd, ret);

	if (ret > 0)
		ret = 0;
	else if (ret < 0)
		ret = -EIO;

474
	blk_mq_complete_request(rq, ret);
Ming Lei's avatar
Ming Lei committed
475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491
}

static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,
		     loff_t pos, bool rw)
{
	struct iov_iter iter;
	struct bio_vec *bvec;
	struct bio *bio = cmd->rq->bio;
	struct file *file = lo->lo_backing_file;
	int ret;

	/* nomerge for loop request queue */
	WARN_ON(cmd->rq->bio != cmd->rq->biotail);

	bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
	iov_iter_bvec(&iter, ITER_BVEC | rw, bvec,
		      bio_segments(bio), blk_rq_bytes(cmd->rq));
492 493 494 495 496 497
	/*
	 * This bio may be started from the middle of the 'bvec'
	 * because of bio splitting, so offset from the bvec must
	 * be passed to iov iterator
	 */
	iter.iov_offset = bio->bi_iter.bi_bvec_done;
Ming Lei's avatar
Ming Lei committed
498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537

	cmd->iocb.ki_pos = pos;
	cmd->iocb.ki_filp = file;
	cmd->iocb.ki_complete = lo_rw_aio_complete;
	cmd->iocb.ki_flags = IOCB_DIRECT;

	if (rw == WRITE)
		ret = file->f_op->write_iter(&cmd->iocb, &iter);
	else
		ret = file->f_op->read_iter(&cmd->iocb, &iter);

	if (ret != -EIOCBQUEUED)
		cmd->iocb.ki_complete(&cmd->iocb, ret, 0);
	return 0;
}


static inline int lo_rw_simple(struct loop_device *lo,
		struct request *rq, loff_t pos, bool rw)
{
	struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);

	if (cmd->use_aio)
		return lo_rw_aio(lo, cmd, pos, rw);

	/*
	 * lo_write_simple and lo_read_simple should have been covered
	 * by io submit style function like lo_rw_aio(), one blocker
	 * is that lo_read_simple() need to call flush_dcache_page after
	 * the page is written from kernel, and it isn't easy to handle
	 * this in io submit style function which submits all segments
	 * of the req at one time. And direct read IO doesn't need to
	 * run flush_dcache_page().
	 */
	if (rw == WRITE)
		return lo_write_simple(lo, rq, pos);
	else
		return lo_read_simple(lo, rq, pos);
}

Ming Lei's avatar
Ming Lei committed
538
static int do_req_filebacked(struct loop_device *lo, struct request *rq)
Linus Torvalds's avatar
Linus Torvalds committed
539 540 541 542
{
	loff_t pos;
	int ret;

Ming Lei's avatar
Ming Lei committed
543
	pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset;
544

Ming Lei's avatar
Ming Lei committed
545
	if (rq->cmd_flags & REQ_WRITE) {
546 547
		if (rq->cmd_flags & REQ_FLUSH)
			ret = lo_req_flush(lo, rq);
548
		else if (rq->cmd_flags & REQ_DISCARD)
549
			ret = lo_discard(lo, rq, pos);
550 551
		else if (lo->transfer)
			ret = lo_write_transfer(lo, rq, pos);
552
		else
Ming Lei's avatar
Ming Lei committed
553
			ret = lo_rw_simple(lo, rq, pos, WRITE);
554 555 556 557 558

	} else {
		if (lo->transfer)
			ret = lo_read_transfer(lo, rq, pos);
		else
Ming Lei's avatar
Ming Lei committed
559
			ret = lo_rw_simple(lo, rq, pos, READ);
560
	}
561

Linus Torvalds's avatar
Linus Torvalds committed
562 563 564 565 566 567 568 569
	return ret;
}

struct switch_request {
	struct file *file;
	struct completion wait;
};

570 571 572 573 574 575
static inline void loop_update_dio(struct loop_device *lo)
{
	__loop_update_dio(lo, io_is_direct(lo->lo_backing_file) |
			lo->use_dio);
}

Linus Torvalds's avatar
Linus Torvalds committed
576
/*
577
 * Do the actual switch; called from the BIO completion routine
Linus Torvalds's avatar
Linus Torvalds committed
578
 */
579
static void do_loop_switch(struct loop_device *lo, struct switch_request *p)
Linus Torvalds's avatar
Linus Torvalds committed
580
{
581 582 583
	struct file *file = p->file;
	struct file *old_file = lo->lo_backing_file;
	struct address_space *mapping;
584

585 586 587
	/* if no new file, only flush of queued bios requested */
	if (!file)
		return;
Linus Torvalds's avatar
Linus Torvalds committed
588

589 590 591 592 593 594 595
	mapping = file->f_mapping;
	mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask);
	lo->lo_backing_file = file;
	lo->lo_blocksize = S_ISBLK(mapping->host->i_mode) ?
		mapping->host->i_bdev->bd_block_size : PAGE_SIZE;
	lo->old_gfp_mask = mapping_gfp_mask(mapping);
	mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
596
	loop_update_dio(lo);
Linus Torvalds's avatar
Linus Torvalds committed
597 598 599 600 601 602 603 604 605 606
}

/*
 * loop_switch performs the hard work of switching a backing store.
 * First it needs to flush existing IO, it does this by sending a magic
 * BIO down the pipe. The completion of this BIO does the actual switch.
 */
static int loop_switch(struct loop_device *lo, struct file *file)
{
	struct switch_request w;
607

Linus Torvalds's avatar
Linus Torvalds committed
608
	w.file = file;
609 610 611 612 613 614 615 616 617 618

	/* freeze queue and wait for completion of scheduled requests */
	blk_mq_freeze_queue(lo->lo_queue);

	/* do the switch action */
	do_loop_switch(lo, &w);

	/* unfreeze */
	blk_mq_unfreeze_queue(lo->lo_queue);

Linus Torvalds's avatar
Linus Torvalds committed
619 620 621
	return 0;
}

622 623 624 625 626
/*
 * Helper to flush the IOs in loop, but keeping loop thread running
 */
static int loop_flush(struct loop_device *lo)
{
627 628 629
	/* loop not yet configured, no running thread, nothing to flush */
	if (lo->lo_state != Lo_bound)
		return 0;
630 631 632
	return loop_switch(lo, NULL);
}

633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654
static void loop_reread_partitions(struct loop_device *lo,
				   struct block_device *bdev)
{
	int rc;

	/*
	 * bd_mutex has been held already in release path, so don't
	 * acquire it if this function is called in such case.
	 *
	 * If the reread partition isn't from release path, lo_refcnt
	 * must be at least one and it can only become zero when the
	 * current holder is released.
	 */
	if (!atomic_read(&lo->lo_refcnt))
		rc = __blkdev_reread_part(bdev);
	else
		rc = blkdev_reread_part(bdev);
	if (rc)
		pr_warn("%s: partition scan of loop%d (%s) failed (rc=%d)\n",
			__func__, lo->lo_number, lo->lo_file_name, rc);
}

655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684
static inline int is_loop_device(struct file *file)
{
	struct inode *i = file->f_mapping->host;

	return i && S_ISBLK(i->i_mode) && MAJOR(i->i_rdev) == LOOP_MAJOR;
}

static int loop_validate_file(struct file *file, struct block_device *bdev)
{
	struct inode	*inode = file->f_mapping->host;
	struct file	*f = file;

	/* Avoid recursion */
	while (is_loop_device(f)) {
		struct loop_device *l;

		if (f->f_mapping->host->i_bdev == bdev)
			return -EBADF;

		l = f->f_mapping->host->i_bdev->bd_disk->private_data;
		if (l->lo_state == Lo_unbound) {
			return -EINVAL;
		}
		f = l->lo_backing_file;
	}
	if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
		return -EINVAL;
	return 0;
}

Linus Torvalds's avatar
Linus Torvalds committed
685 686 687 688 689 690 691 692
/*
 * loop_change_fd switched the backing store of a loopback device to
 * a new file. This is useful for operating system installers to free up
 * the original file and in High Availability environments to switch to
 * an alternative location for the content in case of server meltdown.
 * This can only work if the loop device is used read-only, and if the
 * new backing store is the same size and type as the old backing store.
 */
Al Viro's avatar
Al Viro committed
693 694
static int loop_change_fd(struct loop_device *lo, struct block_device *bdev,
			  unsigned int arg)
Linus Torvalds's avatar
Linus Torvalds committed
695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713
{
	struct file	*file, *old_file;
	struct inode	*inode;
	int		error;

	error = -ENXIO;
	if (lo->lo_state != Lo_bound)
		goto out;

	/* the loop device has to be read-only */
	error = -EINVAL;
	if (!(lo->lo_flags & LO_FLAGS_READ_ONLY))
		goto out;

	error = -EBADF;
	file = fget(arg);
	if (!file)
		goto out;

714 715 716 717
	error = loop_validate_file(file, bdev);
	if (error)
		goto out_putf;

Linus Torvalds's avatar
Linus Torvalds committed
718 719 720 721 722 723 724 725 726 727 728 729 730 731 732
	inode = file->f_mapping->host;
	old_file = lo->lo_backing_file;

	error = -EINVAL;

	/* size of the new backing store needs to be the same */
	if (get_loop_size(lo, file) != get_loop_size(lo, old_file))
		goto out_putf;

	/* and ... switch */
	error = loop_switch(lo, file);
	if (error)
		goto out_putf;

	fput(old_file);
733
	if (lo->lo_flags & LO_FLAGS_PARTSCAN)
734
		loop_reread_partitions(lo, bdev);
Linus Torvalds's avatar
Linus Torvalds committed
735 736 737 738 739 740 741 742
	return 0;

 out_putf:
	fput(file);
 out:
	return error;
}

743 744 745 746 747
/* loop sysfs attributes */

static ssize_t loop_attr_show(struct device *dev, char *page,
			      ssize_t (*callback)(struct loop_device *, char *))
{
748 749
	struct gendisk *disk = dev_to_disk(dev);
	struct loop_device *lo = disk->private_data;
750

751
	return callback(lo, page);
752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768
}

#define LOOP_ATTR_RO(_name)						\
static ssize_t loop_attr_##_name##_show(struct loop_device *, char *);	\
static ssize_t loop_attr_do_show_##_name(struct device *d,		\
				struct device_attribute *attr, char *b)	\
{									\
	return loop_attr_show(d, b, loop_attr_##_name##_show);		\
}									\
static struct device_attribute loop_attr_##_name =			\
	__ATTR(_name, S_IRUGO, loop_attr_do_show_##_name, NULL);

static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf)
{
	ssize_t ret;
	char *p = NULL;

769
	spin_lock_irq(&lo->lo_lock);
770
	if (lo->lo_backing_file)
Miklos Szeredi's avatar
Miklos Szeredi committed
771
		p = file_path(lo->lo_backing_file, buf, PAGE_SIZE - 1);
772
	spin_unlock_irq(&lo->lo_lock);
773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802

	if (IS_ERR_OR_NULL(p))
		ret = PTR_ERR(p);
	else {
		ret = strlen(p);
		memmove(buf, p, ret);
		buf[ret++] = '\n';
		buf[ret] = 0;
	}

	return ret;
}

static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf)
{
	return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_offset);
}

static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf)
{
	return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit);
}

static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf)
{
	int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR);

	return sprintf(buf, "%s\n", autoclear ? "1" : "0");
}

803 804 805 806 807 808 809
static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf)
{
	int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN);

	return sprintf(buf, "%s\n", partscan ? "1" : "0");
}

810 811 812 813 814 815 816
static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf)
{
	int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO);

	return sprintf(buf, "%s\n", dio ? "1" : "0");
}

817 818 819 820
LOOP_ATTR_RO(backing_file);
LOOP_ATTR_RO(offset);
LOOP_ATTR_RO(sizelimit);
LOOP_ATTR_RO(autoclear);
821
LOOP_ATTR_RO(partscan);
822
LOOP_ATTR_RO(dio);
823 824 825 826 827 828

static struct attribute *loop_attrs[] = {
	&loop_attr_backing_file.attr,
	&loop_attr_offset.attr,
	&loop_attr_sizelimit.attr,
	&loop_attr_autoclear.attr,
829
	&loop_attr_partscan.attr,
830
	&loop_attr_dio.attr,
831 832 833 834 835 836 837 838
	NULL,
};

static struct attribute_group loop_attribute_group = {
	.name = "loop",
	.attrs= loop_attrs,
};

839
static void loop_sysfs_init(struct loop_device *lo)
840
{
841 842
	lo->sysfs_inited = !sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj,
						&loop_attribute_group);
843 844 845 846
}

static void loop_sysfs_exit(struct loop_device *lo)
{
847 848 849
	if (lo->sysfs_inited)
		sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj,
				   &loop_attribute_group);
850 851
}

852 853 854 855 856 857 858 859
static void loop_config_discard(struct loop_device *lo)
{
	struct file *file = lo->lo_backing_file;
	struct inode *inode = file->f_mapping->host;
	struct request_queue *q = lo->lo_queue;

	/*
	 * We use punch hole to reclaim the free space used by the
860
	 * image a.k.a. discard. However we do not support discard if
861 862 863 864 865 866 867
	 * encryption is enabled, because it may give an attacker
	 * useful information.
	 */
	if ((!file->f_op->fallocate) ||
	    lo->lo_encrypt_key_size) {
		q->limits.discard_granularity = 0;
		q->limits.discard_alignment = 0;
868
		blk_queue_max_discard_sectors(q, 0);
869 870 871 872 873 874
		q->limits.discard_zeroes_data = 0;
		queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q);
		return;
	}

	q->limits.discard_granularity = inode->i_sb->s_blocksize;
875
	q->limits.discard_alignment = 0;
876
	blk_queue_max_discard_sectors(q, UINT_MAX >> 9);
877 878 879 880
	q->limits.discard_zeroes_data = 1;
	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
}

Ming Lei's avatar
Ming Lei committed
881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897
static void loop_unprepare_queue(struct loop_device *lo)
{
	flush_kthread_worker(&lo->worker);
	kthread_stop(lo->worker_task);
}

static int loop_prepare_queue(struct loop_device *lo)
{
	init_kthread_worker(&lo->worker);
	lo->worker_task = kthread_run(kthread_worker_fn,
			&lo->worker, "loop%d", lo->lo_number);
	if (IS_ERR(lo->worker_task))
		return -ENOMEM;
	set_user_nice(lo->worker_task, MIN_NICE);
	return 0;
}

Al Viro's avatar
Al Viro committed
898
static int loop_set_fd(struct loop_device *lo, fmode_t mode,
Linus Torvalds's avatar
Linus Torvalds committed
899 900
		       struct block_device *bdev, unsigned int arg)
{
901
	struct file	*file;
Linus Torvalds's avatar
Linus Torvalds committed
902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920
	struct inode	*inode;
	struct address_space *mapping;
	unsigned lo_blocksize;
	int		lo_flags = 0;
	int		error;
	loff_t		size;

	/* This is safe, since we have a reference from open(). */
	__module_get(THIS_MODULE);

	error = -EBADF;
	file = fget(arg);
	if (!file)
		goto out;

	error = -EBUSY;
	if (lo->lo_state != Lo_unbound)
		goto out_putf;

921 922 923
	error = loop_validate_file(file, bdev);
	if (error)
		goto out_putf;
Linus Torvalds's avatar
Linus Torvalds committed
924 925 926 927

	mapping = file->f_mapping;
	inode = mapping->host;

928
	if (!(file->f_mode & FMODE_WRITE) || !(mode & FMODE_WRITE) ||
Al Viro's avatar
Al Viro committed
929
	    !file->f_op->write_iter)
930
		lo_flags |= LO_FLAGS_READ_ONLY;
931

932 933
	lo_blocksize = S_ISBLK(inode->i_mode) ?
		inode->i_bdev->bd_block_size : PAGE_SIZE;
Linus Torvalds's avatar
Linus Torvalds committed
934

935
	error = -EFBIG;
Linus Torvalds's avatar
Linus Torvalds committed
936
	size = get_loop_size(lo, file);
937
	if ((loff_t)(sector_t)size != size)
Linus Torvalds's avatar
Linus Torvalds committed
938
		goto out_putf;
Ming Lei's avatar
Ming Lei committed
939 940
	error = loop_prepare_queue(lo);
	if (error)
941
		goto out_putf;
Linus Torvalds's avatar
Linus Torvalds committed
942

943
	error = 0;
Linus Torvalds's avatar
Linus Torvalds committed
944 945 946

	set_device_ro(bdev, (lo_flags & LO_FLAGS_READ_ONLY) != 0);

947
	lo->use_dio = false;
Linus Torvalds's avatar
Linus Torvalds committed
948 949 950 951
	lo->lo_blocksize = lo_blocksize;
	lo->lo_device = bdev;
	lo->lo_flags = lo_flags;
	lo->lo_backing_file = file;
952
	lo->transfer = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
953 954 955 956 957
	lo->ioctl = NULL;
	lo->lo_sizelimit = 0;
	lo->old_gfp_mask = mapping_gfp_mask(mapping);
	mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));

958
	if (!(lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync)
959
		blk_queue_flush(lo->lo_queue, REQ_FLUSH);
960

961
	loop_update_dio(lo);
962
	set_capacity(lo->lo_disk, size);
Linus Torvalds's avatar
Linus Torvalds committed
963
	bd_set_size(bdev, size << 9);
964
	loop_sysfs_init(lo);
965 966
	/* let user-space know about the new size */
	kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
Linus Torvalds's avatar
Linus Torvalds committed
967 968 969

	set_blocksize(bdev, lo_blocksize);

970
	lo->lo_state = Lo_bound;
971 972 973
	if (part_shift)
		lo->lo_flags |= LO_FLAGS_PARTSCAN;
	if (lo->lo_flags & LO_FLAGS_PARTSCAN)
974
		loop_reread_partitions(lo, bdev);
975 976 977 978 979

	/* Grab the block_device to prevent its destruction after we
	 * put /dev/loopXX inode. Later in loop_clr_fd() we bdput(bdev).
	 */
	bdgrab(bdev);
Linus Torvalds's avatar
Linus Torvalds committed
980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026
	return 0;

 out_putf:
	fput(file);
 out:
	/* This is safe: open() is still holding a reference. */
	module_put(THIS_MODULE);
	return error;
}

static int
loop_release_xfer(struct loop_device *lo)
{
	int err = 0;
	struct loop_func_table *xfer = lo->lo_encryption;

	if (xfer) {
		if (xfer->release)
			err = xfer->release(lo);
		lo->transfer = NULL;
		lo->lo_encryption = NULL;
		module_put(xfer->owner);
	}
	return err;
}

static int
loop_init_xfer(struct loop_device *lo, struct loop_func_table *xfer,
	       const struct loop_info64 *i)
{
	int err = 0;

	if (xfer) {
		struct module *owner = xfer->owner;

		if (!try_module_get(owner))
			return -EINVAL;
		if (xfer->init)
			err = xfer->init(lo, i);
		if (err)
			module_put(owner);
		else
			lo->lo_encryption = xfer;
	}
	return err;
}

1027
static int loop_clr_fd(struct loop_device *lo)
Linus Torvalds's avatar
Linus Torvalds committed
1028 1029
{
	struct file *filp = lo->lo_backing_file;
1030
	gfp_t gfp = lo->old_gfp_mask;
1031
	struct block_device *bdev = lo->lo_device;
Linus Torvalds's avatar
Linus Torvalds committed
1032 1033 1034 1035

	if (lo->lo_state != Lo_bound)
		return -ENXIO;

1036 1037 1038 1039 1040 1041 1042 1043 1044 1045
	/*
	 * If we've explicitly asked to tear down the loop device,
	 * and it has an elevated reference count, set it for auto-teardown when
	 * the last reference goes away. This stops $!~#$@ udev from
	 * preventing teardown because it decided that it needs to run blkid on
	 * the loopback device whenever they appear. xfstests is notorious for
	 * failing tests because blkid via udev races with a losetup
	 * <dev>/do something like mkfs/losetup -d <dev> causing the losetup -d
	 * command to fail with EBUSY.
	 */
1046
	if (atomic_read(&lo->lo_refcnt) > 1) {
1047
		lo->lo_flags |= LO_FLAGS_AUTOCLEAR;
1048
		mutex_unlock(&loop_ctl_mutex);
1049 1050
		return 0;
	}
Linus Torvalds's avatar
Linus Torvalds committed
1051 1052 1053 1054

	if (filp == NULL)
		return -EINVAL;

1055 1056 1057
	/* freeze request queue during the transition */
	blk_mq_freeze_queue(lo->lo_queue);

Linus Torvalds's avatar
Linus Torvalds committed
1058 1059 1060
	spin_lock_irq(&lo->lo_lock);
	lo->lo_state = Lo_rundown;
	lo->lo_backing_file = NULL;
1061
	spin_unlock_irq(&lo->lo_lock);
Linus Torvalds's avatar
Linus Torvalds committed
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073

	loop_release_xfer(lo);
	lo->transfer = NULL;
	lo->ioctl = NULL;
	lo->lo_device = NULL;
	lo->lo_encryption = NULL;
	lo->lo_offset = 0;
	lo->lo_sizelimit = 0;
	lo->lo_encrypt_key_size = 0;
	memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE);
	memset(lo->lo_crypt_name, 0, LO_NAME_SIZE);
	memset(lo->lo_file_name, 0, LO_NAME_SIZE);
1074 1075
	if (bdev) {
		bdput(bdev);
Al Viro's avatar
Al Viro committed
1076
		invalidate_bdev(bdev);
1077
	}
1078
	set_capacity(lo->lo_disk, 0);
1079
	loop_sysfs_exit(lo);
1080
	if (bdev) {
Al Viro's avatar
Al Viro committed
1081
		bd_set_size(bdev, 0);
1082 1083 1084
		/* let user-space know about this change */
		kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
	}
Linus Torvalds's avatar
Linus Torvalds committed
1085 1086 1087 1088
	mapping_set_gfp_mask(filp->f_mapping, gfp);
	lo->lo_state = Lo_unbound;
	/* This is safe: open() is still holding a reference. */
	module_put(THIS_MODULE);
1089 1090
	blk_mq_unfreeze_queue(lo->lo_queue);

1091
	if (lo->lo_flags & LO_FLAGS_PARTSCAN && bdev)
1092
		loop_reread_partitions(lo, bdev);
1093 1094 1095
	lo->lo_flags = 0;
	if (!part_shift)
		lo->lo_disk->flags |= GENHD_FL_NO_PART_SCAN;
Ming Lei's avatar
Ming Lei committed
1096
	loop_unprepare_queue(lo);
1097
	mutex_unlock(&loop_ctl_mutex);
1098
	/*
1099 1100
	 * Need not hold loop_ctl_mutex to fput backing file.
	 * Calling fput holding loop_ctl_mutex triggers a circular
1101
	 * lock dependency possibility warning as fput can take
1102
	 * bd_mutex which is usually taken before loop_ctl_mutex.
1103 1104
	 */
	fput(filp);
Linus Torvalds's avatar
Linus Torvalds committed
1105 1106 1107 1108 1109 1110 1111 1112
	return 0;
}

static int
loop_set_status(struct loop_device *lo, const struct loop_info64 *info)
{
	int err;
	struct loop_func_table *xfer;
1113
	kuid_t uid = current_uid();
Linus Torvalds's avatar
Linus Torvalds committed
1114

1115
	if (lo->lo_encrypt_key_size &&
1116
	    !uid_eq(lo->lo_key_owner, uid) &&
Linus Torvalds's avatar
Linus Torvalds committed
1117 1118 1119 1120 1121 1122 1123
	    !capable(CAP_SYS_ADMIN))
		return -EPERM;
	if (lo->lo_state != Lo_bound)
		return -ENXIO;
	if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE)
		return -EINVAL;

1124 1125 1126
	/* I/O need to be drained during transfer transition */
	blk_mq_freeze_queue(lo->lo_queue);

Linus Torvalds's avatar
Linus Torvalds committed
1127 1128
	err = loop_release_xfer(lo);
	if (err)
1129
		goto exit;
Linus Torvalds's avatar
Linus Torvalds committed
1130 1131 1132 1133

	if (info->lo_encrypt_type) {
		unsigned int type = info->lo_encrypt_type;

1134 1135 1136 1137
		if (type >= MAX_LO_CRYPT) {
			err = -EINVAL;
			goto exit;
		}
Linus Torvalds's avatar
Linus Torvalds committed
1138
		xfer = xfer_funcs[type];
1139 1140 1141 1142
		if (xfer == NULL) {
			err = -EINVAL;
			goto exit;
		}
Linus Torvalds's avatar
Linus Torvalds committed
1143 1144 1145 1146 1147
	} else
		xfer = NULL;

	err = loop_init_xfer(lo, xfer, info);
	if (err)
1148
		goto exit;
Linus Torvalds's avatar
Linus Torvalds committed
1149 1150

	if (lo->lo_offset != info->lo_offset ||
1151
	    lo->lo_sizelimit != info->lo_sizelimit)
1152 1153 1154 1155
		if (figure_loop_size(lo, info->lo_offset, info->lo_sizelimit)) {
			err = -EFBIG;
			goto exit;
		}
1156

1157
	loop_config_discard(lo);
Linus Torvalds's avatar
Linus Torvalds committed
1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168

	memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE);
	memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE);
	lo->lo_file_name[LO_NAME_SIZE-1] = 0;
	lo->lo_crypt_name[LO_NAME_SIZE-1] = 0;

	if (!xfer)
		xfer = &none_funcs;
	lo->transfer = xfer->transfer;
	lo->ioctl = xfer->ioctl;

1169 1170 1171 1172
	if ((lo->lo_flags & LO_FLAGS_AUTOCLEAR) !=
	     (info->lo_flags & LO_FLAGS_AUTOCLEAR))
		lo->lo_flags ^= LO_FLAGS_AUTOCLEAR;

Linus Torvalds's avatar
Linus Torvalds committed
1173 1174 1175 1176 1177 1178
	lo->lo_encrypt_key_size = info->lo_encrypt_key_size;
	lo->lo_init[0] = info->lo_init[0];
	lo->lo_init[1] = info->lo_init[1];
	if (info->lo_encrypt_key_size) {
		memcpy(lo->lo_encrypt_key, info->lo_encrypt_key,
		       info->lo_encrypt_key_size);
1179
		lo->lo_key_owner = uid;
1180
	}
Linus Torvalds's avatar
Linus Torvalds committed
1181

1182 1183 1184
	/* update dio if lo_offset or transfer is changed */
	__loop_update_dio(lo, lo->use_dio);

1185 1186
 exit:
	blk_mq_unfreeze_queue(lo->lo_queue);
1187 1188 1189 1190 1191 1192 1193 1194

	if (!err && (info->lo_flags & LO_FLAGS_PARTSCAN) &&
	     !(lo->lo_flags & LO_FLAGS_PARTSCAN)) {
		lo->lo_flags |= LO_FLAGS_PARTSCAN;
		lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN;
		loop_reread_partitions(lo, lo->lo_device);
	}

1195
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206
}

static int
loop_get_status(struct loop_device *lo, struct loop_info64 *info)
{
	struct file *file = lo->lo_backing_file;
	struct kstat stat;
	int error;

	if (lo->lo_state != Lo_bound)
		return -ENXIO;
Al Viro's avatar
Al Viro committed
1207
	error = vfs_getattr(&file->f_path, &stat);
Linus Torvalds's avatar
Linus Torvalds committed
1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336
	if (error)
		return error;
	memset(info, 0, sizeof(*info));
	info->lo_number = lo->lo_number;
	info->lo_device = huge_encode_dev(stat.dev);
	info->lo_inode = stat.ino;
	info->lo_rdevice = huge_encode_dev(lo->lo_device ? stat.rdev : stat.dev);
	info->lo_offset = lo->lo_offset;
	info->lo_sizelimit = lo->lo_sizelimit;
	info->lo_flags = lo->lo_flags;
	memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE);
	memcpy(info->lo_crypt_name, lo->lo_crypt_name, LO_NAME_SIZE);
	info->lo_encrypt_type =
		lo->lo_encryption ? lo->lo_encryption->number : 0;
	if (lo->lo_encrypt_key_size && capable(CAP_SYS_ADMIN)) {
		info->lo_encrypt_key_size = lo->lo_encrypt_key_size;
		memcpy(info->lo_encrypt_key, lo->lo_encrypt_key,
		       lo->lo_encrypt_key_size);
	}
	return 0;
}

static void
loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64)
{
	memset(info64, 0, sizeof(*info64));
	info64->lo_number = info->lo_number;
	info64->lo_device = info->lo_device;
	info64->lo_inode = info->lo_inode;
	info64->lo_rdevice = info->lo_rdevice;
	info64->lo_offset = info->lo_offset;
	info64->lo_sizelimit = 0;
	info64->lo_encrypt_type = info->lo_encrypt_type;
	info64->lo_encrypt_key_size = info->lo_encrypt_key_size;
	info64->lo_flags = info->lo_flags;
	info64->lo_init[0] = info->lo_init[0];
	info64->lo_init[1] = info->lo_init[1];
	if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
		memcpy(info64->lo_crypt_name, info->lo_name, LO_NAME_SIZE);
	else
		memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE);
	memcpy(info64->lo_encrypt_key, info->lo_encrypt_key, LO_KEY_SIZE);
}

static int
loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info)
{
	memset(info, 0, sizeof(*info));
	info->lo_number = info64->lo_number;
	info->lo_device = info64->lo_device;
	info->lo_inode = info64->lo_inode;
	info->lo_rdevice = info64->lo_rdevice;
	info->lo_offset = info64->lo_offset;
	info->lo_encrypt_type = info64->lo_encrypt_type;
	info->lo_encrypt_key_size = info64->lo_encrypt_key_size;
	info->lo_flags = info64->lo_flags;
	info->lo_init[0] = info64->lo_init[0];
	info->lo_init[1] = info64->lo_init[1];
	if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
		memcpy(info->lo_name, info64->lo_crypt_name, LO_NAME_SIZE);
	else
		memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE);
	memcpy(info->lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE);

	/* error in case values were truncated */
	if (info->lo_device != info64->lo_device ||
	    info->lo_rdevice != info64->lo_rdevice ||
	    info->lo_inode != info64->lo_inode ||
	    info->lo_offset != info64->lo_offset)
		return -EOVERFLOW;

	return 0;
}

static int
loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg)
{
	struct loop_info info;
	struct loop_info64 info64;

	if (copy_from_user(&info, arg, sizeof (struct loop_info)))
		return -EFAULT;
	loop_info64_from_old(&info, &info64);
	return loop_set_status(lo, &info64);
}

static int
loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg)
{
	struct loop_info64 info64;

	if (copy_from_user(&info64, arg, sizeof (struct loop_info64)))
		return -EFAULT;
	return loop_set_status(lo, &info64);
}

static int
loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) {
	struct loop_info info;
	struct loop_info64 info64;
	int err = 0;

	if (!arg)
		err = -EINVAL;
	if (!err)
		err = loop_get_status(lo, &info64);
	if (!err)
		err = loop_info64_to_old(&info64, &info);
	if (!err && copy_to_user(arg, &info, sizeof(info)))
		err = -EFAULT;

	return err;
}

static int
loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) {
	struct loop_info64 info64;
	int err = 0;

	if (!arg)
		err = -EINVAL;
	if (!err)
		err = loop_get_status(lo, &info64);
	if (!err && copy_to_user(arg, &info64, sizeof(info64)))
		err = -EFAULT;

	return err;
}

1337 1338 1339
static int loop_set_capacity(struct loop_device *lo, struct block_device *bdev)
{
	if (unlikely(lo->lo_state != Lo_bound))
1340
		return -ENXIO;
1341

1342
	return figure_loop_size(lo, lo->lo_offset, lo->lo_sizelimit);
1343 1344
}

1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358
static int loop_set_dio(struct loop_device *lo, unsigned long arg)
{
	int error = -ENXIO;
	if (lo->lo_state != Lo_bound)
		goto out;

	__loop_update_dio(lo, !!arg);
	if (lo->use_dio == !!arg)
		return 0;
	error = -EINVAL;
 out:
	return error;
}

Al Viro's avatar
Al Viro committed
1359
static int lo_ioctl(struct block_device *bdev, fmode_t mode,
Linus Torvalds's avatar
Linus Torvalds committed
1360 1361
	unsigned int cmd, unsigned long arg)
{
Al Viro's avatar
Al Viro committed
1362
	struct loop_device *lo = bdev->bd_disk->private_data;
Linus Torvalds's avatar
Linus Torvalds committed
1363 1364
	int err;