bio.c 48.4 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
/*
2
 * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
Linus Torvalds's avatar
Linus Torvalds committed
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public Licens
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-
 *
 */
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
22
#include <linux/uio.h>
23
#include <linux/iocontext.h>
Linus Torvalds's avatar
Linus Torvalds committed
24 25 26
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/kernel.h>
27
#include <linux/export.h>
Linus Torvalds's avatar
Linus Torvalds committed
28 29
#include <linux/mempool.h>
#include <linux/workqueue.h>
30
#include <linux/cgroup.h>
Linus Torvalds's avatar
Linus Torvalds committed
31

32
#include <trace/events/block.h>
33

34 35 36 37 38 39
/*
 * Test patch to inline a certain number of bi_io_vec's inside the bio
 * itself, to shrink a bio data allocation from two mempool calls to one
 */
#define BIO_INLINE_VECS		4

Linus Torvalds's avatar
Linus Torvalds committed
40 41 42 43 44 45
/*
 * if you change this list, also change bvec_alloc or things will
 * break badly! cannot be bigger than what you can fit into an
 * unsigned short
 */
#define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) }
46
static struct biovec_slab bvec_slabs[BVEC_POOL_NR] __read_mostly = {
Linus Torvalds's avatar
Linus Torvalds committed
47 48 49 50 51 52 53 54
	BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES),
};
#undef BV

/*
 * fs_bio_set is the bio_set containing bio and iovec memory pools used by
 * IO code that does not need private memory pools.
 */
55
struct bio_set *fs_bio_set;
56
EXPORT_SYMBOL(fs_bio_set);
Linus Torvalds's avatar
Linus Torvalds committed
57

58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74
/*
 * Our slab pool management
 */
struct bio_slab {
	struct kmem_cache *slab;
	unsigned int slab_ref;
	unsigned int slab_size;
	char name[8];
};
static DEFINE_MUTEX(bio_slab_lock);
static struct bio_slab *bio_slabs;
static unsigned int bio_slab_nr, bio_slab_max;

static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size)
{
	unsigned int sz = sizeof(struct bio) + extra_size;
	struct kmem_cache *slab = NULL;
75
	struct bio_slab *bslab, *new_bio_slabs;
76
	unsigned int new_bio_slab_max;
77 78 79 80 81 82
	unsigned int i, entry = -1;

	mutex_lock(&bio_slab_lock);

	i = 0;
	while (i < bio_slab_nr) {
83
		bslab = &bio_slabs[i];
84 85 86 87 88 89 90 91 92 93 94 95 96 97 98

		if (!bslab->slab && entry == -1)
			entry = i;
		else if (bslab->slab_size == sz) {
			slab = bslab->slab;
			bslab->slab_ref++;
			break;
		}
		i++;
	}

	if (slab)
		goto out_unlock;

	if (bio_slab_nr == bio_slab_max && entry == -1) {
99
		new_bio_slab_max = bio_slab_max << 1;
100
		new_bio_slabs = krealloc(bio_slabs,
101
					 new_bio_slab_max * sizeof(struct bio_slab),
102 103
					 GFP_KERNEL);
		if (!new_bio_slabs)
104
			goto out_unlock;
105
		bio_slab_max = new_bio_slab_max;
106
		bio_slabs = new_bio_slabs;
107 108 109 110 111 112 113
	}
	if (entry == -1)
		entry = bio_slab_nr++;

	bslab = &bio_slabs[entry];

	snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry);
114 115
	slab = kmem_cache_create(bslab->name, sz, ARCH_KMALLOC_MINALIGN,
				 SLAB_HWCACHE_ALIGN, NULL);
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
	if (!slab)
		goto out_unlock;

	bslab->slab = slab;
	bslab->slab_ref = 1;
	bslab->slab_size = sz;
out_unlock:
	mutex_unlock(&bio_slab_lock);
	return slab;
}

static void bio_put_slab(struct bio_set *bs)
{
	struct bio_slab *bslab = NULL;
	unsigned int i;

	mutex_lock(&bio_slab_lock);

	for (i = 0; i < bio_slab_nr; i++) {
		if (bs->bio_slab == bio_slabs[i].slab) {
			bslab = &bio_slabs[i];
			break;
		}
	}

	if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))
		goto out;

	WARN_ON(!bslab->slab_ref);

	if (--bslab->slab_ref)
		goto out;

	kmem_cache_destroy(bslab->slab);
	bslab->slab = NULL;

out:
	mutex_unlock(&bio_slab_lock);
}

156 157 158 159 160
unsigned int bvec_nr_vecs(unsigned short idx)
{
	return bvec_slabs[idx].nr_vecs;
}

161
void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
162
{
163 164 165 166 167
	if (!idx)
		return;
	idx--;

	BIO_BUG_ON(idx >= BVEC_POOL_NR);
168

169
	if (idx == BVEC_POOL_MAX) {
170
		mempool_free(bv, pool);
171
	} else {
172 173 174 175 176 177
		struct biovec_slab *bvs = bvec_slabs + idx;

		kmem_cache_free(bvs->slab, bv);
	}
}

178 179
struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
			   mempool_t *pool)
Linus Torvalds's avatar
Linus Torvalds committed
180 181 182
{
	struct bio_vec *bvl;

183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
	/*
	 * see comment near bvec_array define!
	 */
	switch (nr) {
	case 1:
		*idx = 0;
		break;
	case 2 ... 4:
		*idx = 1;
		break;
	case 5 ... 16:
		*idx = 2;
		break;
	case 17 ... 64:
		*idx = 3;
		break;
	case 65 ... 128:
		*idx = 4;
		break;
	case 129 ... BIO_MAX_PAGES:
		*idx = 5;
		break;
	default:
		return NULL;
	}

	/*
	 * idx now points to the pool we want to allocate from. only the
	 * 1-vec entry pool is mempool backed.
	 */
213
	if (*idx == BVEC_POOL_MAX) {
214
fallback:
215
		bvl = mempool_alloc(pool, gfp_mask);
216 217
	} else {
		struct biovec_slab *bvs = bvec_slabs + *idx;
218
		gfp_t __gfp_mask = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_IO);
219

Jens Axboe's avatar
Jens Axboe committed
220
		/*
221 222 223
		 * Make this allocation restricted and don't dump info on
		 * allocation failures, since we'll fallback to the mempool
		 * in case of failure.
Jens Axboe's avatar
Jens Axboe committed
224
		 */
225
		__gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
Linus Torvalds's avatar
Linus Torvalds committed
226

Jens Axboe's avatar
Jens Axboe committed
227
		/*
228
		 * Try a slab allocation. If this fails and __GFP_DIRECT_RECLAIM
229
		 * is set, retry with the 1-entry mempool
Jens Axboe's avatar
Jens Axboe committed
230
		 */
231
		bvl = kmem_cache_alloc(bvs->slab, __gfp_mask);
232
		if (unlikely(!bvl && (gfp_mask & __GFP_DIRECT_RECLAIM))) {
233
			*idx = BVEC_POOL_MAX;
234 235 236 237
			goto fallback;
		}
	}

238
	(*idx)++;
Linus Torvalds's avatar
Linus Torvalds committed
239 240 241
	return bvl;
}

Kent Overstreet's avatar
Kent Overstreet committed
242
static void __bio_free(struct bio *bio)
Linus Torvalds's avatar
Linus Torvalds committed
243
{
Kent Overstreet's avatar
Kent Overstreet committed
244
	bio_disassociate_task(bio);
Linus Torvalds's avatar
Linus Torvalds committed
245

246
	if (bio_integrity(bio))
247
		bio_integrity_free(bio);
Kent Overstreet's avatar
Kent Overstreet committed
248
}
249

Kent Overstreet's avatar
Kent Overstreet committed
250 251 252 253 254 255 256 257
static void bio_free(struct bio *bio)
{
	struct bio_set *bs = bio->bi_pool;
	void *p;

	__bio_free(bio);

	if (bs) {
258
		bvec_free(bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));
Kent Overstreet's avatar
Kent Overstreet committed
259 260 261 262 263

		/*
		 * If we have front padding, adjust the bio pointer before freeing
		 */
		p = bio;
264 265
		p -= bs->front_pad;

Kent Overstreet's avatar
Kent Overstreet committed
266 267 268 269 270
		mempool_free(p, bs->bio_pool);
	} else {
		/* Bio was allocated by bio_kmalloc() */
		kfree(bio);
	}
Peter Osterlund's avatar
Peter Osterlund committed
271 272
}

273
void bio_init(struct bio *bio)
Linus Torvalds's avatar
Linus Torvalds committed
274
{
Jens Axboe's avatar
Jens Axboe committed
275
	memset(bio, 0, sizeof(*bio));
276
	atomic_set(&bio->__bi_remaining, 1);
277
	atomic_set(&bio->__bi_cnt, 1);
Linus Torvalds's avatar
Linus Torvalds committed
278
}
279
EXPORT_SYMBOL(bio_init);
Linus Torvalds's avatar
Linus Torvalds committed
280

Kent Overstreet's avatar
Kent Overstreet committed
281 282 283 284 285 286 287 288 289 290 291 292 293 294
/**
 * bio_reset - reinitialize a bio
 * @bio:	bio to reset
 *
 * Description:
 *   After calling bio_reset(), @bio will be in the same state as a freshly
 *   allocated bio returned bio bio_alloc_bioset() - the only fields that are
 *   preserved are the ones that are initialized by bio_alloc_bioset(). See
 *   comment in struct bio.
 */
void bio_reset(struct bio *bio)
{
	unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);

Kent Overstreet's avatar
Kent Overstreet committed
295
	__bio_free(bio);
Kent Overstreet's avatar
Kent Overstreet committed
296 297

	memset(bio, 0, BIO_RESET_BYTES);
298
	bio->bi_flags = flags;
299
	atomic_set(&bio->__bi_remaining, 1);
Kent Overstreet's avatar
Kent Overstreet committed
300 301 302
}
EXPORT_SYMBOL(bio_reset);

303
static struct bio *__bio_chain_endio(struct bio *bio)
Kent Overstreet's avatar
Kent Overstreet committed
304
{
305 306
	struct bio *parent = bio->bi_private;

307 308
	if (!parent->bi_error)
		parent->bi_error = bio->bi_error;
Kent Overstreet's avatar
Kent Overstreet committed
309
	bio_put(bio);
310 311 312 313 314 315
	return parent;
}

static void bio_chain_endio(struct bio *bio)
{
	bio_endio(__bio_chain_endio(bio));
Kent Overstreet's avatar
Kent Overstreet committed
316 317 318 319
}

/**
 * bio_chain - chain bio completions
320 321
 * @bio: the target bio
 * @parent: the @bio's parent bio
Kent Overstreet's avatar
Kent Overstreet committed
322 323 324 325 326 327 328 329 330 331 332 333 334
 *
 * The caller won't have a bi_end_io called when @bio completes - instead,
 * @parent's bi_end_io won't be called until both @parent and @bio have
 * completed; the chained bio will also be freed when it completes.
 *
 * The caller must not set bi_private or bi_end_io in @bio.
 */
void bio_chain(struct bio *bio, struct bio *parent)
{
	BUG_ON(bio->bi_private || bio->bi_end_io);

	bio->bi_private = parent;
	bio->bi_end_io	= bio_chain_endio;
335
	bio_inc_remaining(parent);
Kent Overstreet's avatar
Kent Overstreet committed
336 337 338
}
EXPORT_SYMBOL(bio_chain);

339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386
static void bio_alloc_rescue(struct work_struct *work)
{
	struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
	struct bio *bio;

	while (1) {
		spin_lock(&bs->rescue_lock);
		bio = bio_list_pop(&bs->rescue_list);
		spin_unlock(&bs->rescue_lock);

		if (!bio)
			break;

		generic_make_request(bio);
	}
}

static void punt_bios_to_rescuer(struct bio_set *bs)
{
	struct bio_list punt, nopunt;
	struct bio *bio;

	/*
	 * In order to guarantee forward progress we must punt only bios that
	 * were allocated from this bio_set; otherwise, if there was a bio on
	 * there for a stacking driver higher up in the stack, processing it
	 * could require allocating bios from this bio_set, and doing that from
	 * our own rescuer would be bad.
	 *
	 * Since bio lists are singly linked, pop them all instead of trying to
	 * remove from the middle of the list:
	 */

	bio_list_init(&punt);
	bio_list_init(&nopunt);

	while ((bio = bio_list_pop(current->bio_list)))
		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);

	*current->bio_list = nopunt;

	spin_lock(&bs->rescue_lock);
	bio_list_merge(&bs->rescue_list, &punt);
	spin_unlock(&bs->rescue_lock);

	queue_work(bs->rescue_workqueue, &bs->rescue_work);
}

Linus Torvalds's avatar
Linus Torvalds committed
387 388 389 390
/**
 * bio_alloc_bioset - allocate a bio for I/O
 * @gfp_mask:   the GFP_ mask given to the slab allocator
 * @nr_iovecs:	number of iovecs to pre-allocate
391
 * @bs:		the bio_set to allocate from.
Linus Torvalds's avatar
Linus Torvalds committed
392 393
 *
 * Description:
394 395 396
 *   If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is
 *   backed by the @bs's mempool.
 *
397 398 399 400 401 402
 *   When @bs is not NULL, if %__GFP_DIRECT_RECLAIM is set then bio_alloc will
 *   always be able to allocate a bio. This is due to the mempool guarantees.
 *   To make this work, callers must never allocate more than 1 bio at a time
 *   from this pool. Callers that need to allocate more than 1 bio must always
 *   submit the previously allocated bio for IO before attempting to allocate
 *   a new one. Failure to do so can cause deadlocks under memory pressure.
403
 *
404 405 406 407 408 409 410 411 412 413 414 415 416 417 418
 *   Note that when running under generic_make_request() (i.e. any block
 *   driver), bios are not submitted until after you return - see the code in
 *   generic_make_request() that converts recursion into iteration, to prevent
 *   stack overflows.
 *
 *   This would normally mean allocating multiple bios under
 *   generic_make_request() would be susceptible to deadlocks, but we have
 *   deadlock avoidance code that resubmits any blocked bios from a rescuer
 *   thread.
 *
 *   However, we do not guarantee forward progress for allocations from other
 *   mempools. Doing multiple allocations from the same mempool under
 *   generic_make_request() should be avoided - instead, use bio_set's front_pad
 *   for per bio allocations.
 *
419 420 421
 *   RETURNS:
 *   Pointer to new bio on success, NULL on failure.
 */
422
struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
Linus Torvalds's avatar
Linus Torvalds committed
423
{
424
	gfp_t saved_gfp = gfp_mask;
425 426
	unsigned front_pad;
	unsigned inline_vecs;
427
	struct bio_vec *bvl = NULL;
Tejun Heo's avatar
Tejun Heo committed
428 429 430
	struct bio *bio;
	void *p;

431 432 433 434 435 436 437 438 439 440
	if (!bs) {
		if (nr_iovecs > UIO_MAXIOV)
			return NULL;

		p = kmalloc(sizeof(struct bio) +
			    nr_iovecs * sizeof(struct bio_vec),
			    gfp_mask);
		front_pad = 0;
		inline_vecs = nr_iovecs;
	} else {
441 442 443
		/* should not use nobvec bioset for nr_iovecs > 0 */
		if (WARN_ON_ONCE(!bs->bvec_pool && nr_iovecs > 0))
			return NULL;
444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459
		/*
		 * generic_make_request() converts recursion to iteration; this
		 * means if we're running beneath it, any bios we allocate and
		 * submit will not be submitted (and thus freed) until after we
		 * return.
		 *
		 * This exposes us to a potential deadlock if we allocate
		 * multiple bios from the same bio_set() while running
		 * underneath generic_make_request(). If we were to allocate
		 * multiple bios (say a stacking block driver that was splitting
		 * bios), we would deadlock if we exhausted the mempool's
		 * reserve.
		 *
		 * We solve this, and guarantee forward progress, with a rescuer
		 * workqueue per bio_set. If we go to allocate and there are
		 * bios on current->bio_list, we first try the allocation
460 461 462
		 * without __GFP_DIRECT_RECLAIM; if that fails, we punt those
		 * bios we would be blocking to the rescuer workqueue before
		 * we retry with the original gfp_flags.
463 464 465
		 */

		if (current->bio_list && !bio_list_empty(current->bio_list))
466
			gfp_mask &= ~__GFP_DIRECT_RECLAIM;
467

468
		p = mempool_alloc(bs->bio_pool, gfp_mask);
469 470 471 472 473 474
		if (!p && gfp_mask != saved_gfp) {
			punt_bios_to_rescuer(bs);
			gfp_mask = saved_gfp;
			p = mempool_alloc(bs->bio_pool, gfp_mask);
		}

475 476 477 478
		front_pad = bs->front_pad;
		inline_vecs = BIO_INLINE_VECS;
	}

Tejun Heo's avatar
Tejun Heo committed
479 480
	if (unlikely(!p))
		return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
481

482
	bio = p + front_pad;
483 484
	bio_init(bio);

485
	if (nr_iovecs > inline_vecs) {
486 487
		unsigned long idx = 0;

488
		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
489 490 491
		if (!bvl && gfp_mask != saved_gfp) {
			punt_bios_to_rescuer(bs);
			gfp_mask = saved_gfp;
492
			bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
493 494
		}

495 496
		if (unlikely(!bvl))
			goto err_free;
497

498
		bio->bi_flags |= idx << BVEC_POOL_OFFSET;
499 500
	} else if (nr_iovecs) {
		bvl = bio->bi_inline_vecs;
Linus Torvalds's avatar
Linus Torvalds committed
501
	}
502 503

	bio->bi_pool = bs;
504 505
	bio->bi_max_vecs = nr_iovecs;
	bio->bi_io_vec = bvl;
Linus Torvalds's avatar
Linus Torvalds committed
506
	return bio;
507 508

err_free:
Tejun Heo's avatar
Tejun Heo committed
509
	mempool_free(p, bs->bio_pool);
510
	return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
511
}
512
EXPORT_SYMBOL(bio_alloc_bioset);
Linus Torvalds's avatar
Linus Torvalds committed
513 514 515 516

void zero_fill_bio(struct bio *bio)
{
	unsigned long flags;
517 518
	struct bio_vec bv;
	struct bvec_iter iter;
Linus Torvalds's avatar
Linus Torvalds committed
519

520 521 522 523
	bio_for_each_segment(bv, bio, iter) {
		char *data = bvec_kmap_irq(&bv, &flags);
		memset(data, 0, bv.bv_len);
		flush_dcache_page(bv.bv_page);
Linus Torvalds's avatar
Linus Torvalds committed
524 525 526 527 528 529 530 531 532 533 534
		bvec_kunmap_irq(data, &flags);
	}
}
EXPORT_SYMBOL(zero_fill_bio);

/**
 * bio_put - release a reference to a bio
 * @bio:   bio to release reference to
 *
 * Description:
 *   Put a reference to a &struct bio, either one you have gotten with
535
 *   bio_alloc, bio_get or bio_clone. The last put of a bio will free it.
Linus Torvalds's avatar
Linus Torvalds committed
536 537 538
 **/
void bio_put(struct bio *bio)
{
539
	if (!bio_flagged(bio, BIO_REFFED))
Kent Overstreet's avatar
Kent Overstreet committed
540
		bio_free(bio);
541 542 543 544 545 546 547 548 549
	else {
		BIO_BUG_ON(!atomic_read(&bio->__bi_cnt));

		/*
		 * last put frees it
		 */
		if (atomic_dec_and_test(&bio->__bi_cnt))
			bio_free(bio);
	}
Linus Torvalds's avatar
Linus Torvalds committed
550
}
551
EXPORT_SYMBOL(bio_put);
Linus Torvalds's avatar
Linus Torvalds committed
552

553
inline int bio_phys_segments(struct request_queue *q, struct bio *bio)
Linus Torvalds's avatar
Linus Torvalds committed
554 555 556 557 558 559
{
	if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
		blk_recount_segments(q, bio);

	return bio->bi_phys_segments;
}
560
EXPORT_SYMBOL(bio_phys_segments);
Linus Torvalds's avatar
Linus Torvalds committed
561

Kent Overstreet's avatar
Kent Overstreet committed
562 563 564 565 566 567 568 569 570 571 572 573 574
/**
 * 	__bio_clone_fast - clone a bio that shares the original bio's biovec
 * 	@bio: destination bio
 * 	@bio_src: bio to clone
 *
 *	Clone a &bio. Caller will own the returned bio, but not
 *	the actual data it points to. Reference count of returned
 * 	bio will be one.
 *
 * 	Caller must ensure that @bio_src is not freed before @bio.
 */
void __bio_clone_fast(struct bio *bio, struct bio *bio_src)
{
575
	BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio));
Kent Overstreet's avatar
Kent Overstreet committed
576 577 578 579 580 581

	/*
	 * most users will be overriding ->bi_bdev with a new target,
	 * so we don't set nor calculate new physical/hw segment counts here
	 */
	bio->bi_bdev = bio_src->bi_bdev;
582
	bio_set_flag(bio, BIO_CLONED);
Kent Overstreet's avatar
Kent Overstreet committed
583 584 585
	bio->bi_rw = bio_src->bi_rw;
	bio->bi_iter = bio_src->bi_iter;
	bio->bi_io_vec = bio_src->bi_io_vec;
586 587

	bio_clone_blkcg_association(bio, bio_src);
Kent Overstreet's avatar
Kent Overstreet committed
588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623
}
EXPORT_SYMBOL(__bio_clone_fast);

/**
 *	bio_clone_fast - clone a bio that shares the original bio's biovec
 *	@bio: bio to clone
 *	@gfp_mask: allocation priority
 *	@bs: bio_set to allocate from
 *
 * 	Like __bio_clone_fast, only also allocates the returned bio
 */
struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)
{
	struct bio *b;

	b = bio_alloc_bioset(gfp_mask, 0, bs);
	if (!b)
		return NULL;

	__bio_clone_fast(b, bio);

	if (bio_integrity(bio)) {
		int ret;

		ret = bio_integrity_clone(b, bio, gfp_mask);

		if (ret < 0) {
			bio_put(b);
			return NULL;
		}
	}

	return b;
}
EXPORT_SYMBOL(bio_clone_fast);

Linus Torvalds's avatar
Linus Torvalds committed
624
/**
625 626
 * 	bio_clone_bioset - clone a bio
 * 	@bio_src: bio to clone
Linus Torvalds's avatar
Linus Torvalds committed
627
 *	@gfp_mask: allocation priority
628
 *	@bs: bio_set to allocate from
Linus Torvalds's avatar
Linus Torvalds committed
629
 *
630 631
 *	Clone bio. Caller will own the returned bio, but not the actual data it
 *	points to. Reference count of returned bio will be one.
Linus Torvalds's avatar
Linus Torvalds committed
632
 */
633
struct bio *bio_clone_bioset(struct bio *bio_src, gfp_t gfp_mask,
634
			     struct bio_set *bs)
Linus Torvalds's avatar
Linus Torvalds committed
635
{
636 637 638
	struct bvec_iter iter;
	struct bio_vec bv;
	struct bio *bio;
Linus Torvalds's avatar
Linus Torvalds committed
639

640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661
	/*
	 * Pre immutable biovecs, __bio_clone() used to just do a memcpy from
	 * bio_src->bi_io_vec to bio->bi_io_vec.
	 *
	 * We can't do that anymore, because:
	 *
	 *  - The point of cloning the biovec is to produce a bio with a biovec
	 *    the caller can modify: bi_idx and bi_bvec_done should be 0.
	 *
	 *  - The original bio could've had more than BIO_MAX_PAGES biovecs; if
	 *    we tried to clone the whole thing bio_alloc_bioset() would fail.
	 *    But the clone should succeed as long as the number of biovecs we
	 *    actually need to allocate is fewer than BIO_MAX_PAGES.
	 *
	 *  - Lastly, bi_vcnt should not be looked at or relied upon by code
	 *    that does not own the bio - reason being drivers don't use it for
	 *    iterating over the biovec anymore, so expecting it to be kept up
	 *    to date (i.e. for clones that share the parent biovec) is just
	 *    asking for trouble and would force extra work on
	 *    __bio_clone_fast() anyways.
	 */

662
	bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs);
663
	if (!bio)
664
		return NULL;
665 666 667 668
	bio->bi_bdev		= bio_src->bi_bdev;
	bio->bi_rw		= bio_src->bi_rw;
	bio->bi_iter.bi_sector	= bio_src->bi_iter.bi_sector;
	bio->bi_iter.bi_size	= bio_src->bi_iter.bi_size;
669

670
	if (bio_op(bio) == REQ_OP_DISCARD)
671 672
		goto integrity_clone;

673
	if (bio_op(bio) == REQ_OP_WRITE_SAME) {
674 675 676 677
		bio->bi_io_vec[bio->bi_vcnt++] = bio_src->bi_io_vec[0];
		goto integrity_clone;
	}

678 679
	bio_for_each_segment(bv, bio_src, iter)
		bio->bi_io_vec[bio->bi_vcnt++] = bv;
680

681
integrity_clone:
682 683
	if (bio_integrity(bio_src)) {
		int ret;
684

685
		ret = bio_integrity_clone(bio, bio_src, gfp_mask);
686
		if (ret < 0) {
687
			bio_put(bio);
688
			return NULL;
689
		}
Peter Osterlund's avatar
Peter Osterlund committed
690
	}
Linus Torvalds's avatar
Linus Torvalds committed
691

692 693
	bio_clone_blkcg_association(bio, bio_src);

694
	return bio;
Linus Torvalds's avatar
Linus Torvalds committed
695
}
696
EXPORT_SYMBOL(bio_clone_bioset);
Linus Torvalds's avatar
Linus Torvalds committed
697 698

/**
699 700 701 702 703 704
 *	bio_add_pc_page	-	attempt to add page to bio
 *	@q: the target queue
 *	@bio: destination bio
 *	@page: page to add
 *	@len: vec entry length
 *	@offset: vec entry offset
Linus Torvalds's avatar
Linus Torvalds committed
705
 *
706 707 708 709 710 711
 *	Attempt to add a page to the bio_vec maplist. This can fail for a
 *	number of reasons, such as the bio being full or target block device
 *	limitations. The target block device must allow bio's up to PAGE_SIZE,
 *	so it is always possible to add a single page to an empty bio.
 *
 *	This should only be used by REQ_PC bios.
Linus Torvalds's avatar
Linus Torvalds committed
712
 */
713 714
int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page
		    *page, unsigned int len, unsigned int offset)
Linus Torvalds's avatar
Linus Torvalds committed
715 716 717 718 719 720 721 722 723 724
{
	int retried_segments = 0;
	struct bio_vec *bvec;

	/*
	 * cloned bio must not modify vec list
	 */
	if (unlikely(bio_flagged(bio, BIO_CLONED)))
		return 0;

725
	if (((bio->bi_iter.bi_size + len) >> 9) > queue_max_hw_sectors(q))
Linus Torvalds's avatar
Linus Torvalds committed
726 727
		return 0;

728 729 730 731 732 733 734 735 736 737 738
	/*
	 * For filesystems with a blocksize smaller than the pagesize
	 * we will often be called with the same page as last time and
	 * a consecutive offset.  Optimize this special case.
	 */
	if (bio->bi_vcnt > 0) {
		struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1];

		if (page == prev->bv_page &&
		    offset == prev->bv_offset + prev->bv_len) {
			prev->bv_len += len;
739
			bio->bi_iter.bi_size += len;
740 741
			goto done;
		}
742 743 744 745 746

		/*
		 * If the queue doesn't support SG gaps and adding this
		 * offset would create a gap, disallow it.
		 */
747
		if (bvec_gap_to_prev(q, prev, offset))
748
			return 0;
749 750 751
	}

	if (bio->bi_vcnt >= bio->bi_max_vecs)
Linus Torvalds's avatar
Linus Torvalds committed
752 753 754
		return 0;

	/*
755 756 757 758 759 760 761 762 763 764 765 766 767 768
	 * setup the new entry, we might clear it again later if we
	 * cannot add the page
	 */
	bvec = &bio->bi_io_vec[bio->bi_vcnt];
	bvec->bv_page = page;
	bvec->bv_len = len;
	bvec->bv_offset = offset;
	bio->bi_vcnt++;
	bio->bi_phys_segments++;
	bio->bi_iter.bi_size += len;

	/*
	 * Perform a recount if the number of segments is greater
	 * than queue_max_segments(q).
Linus Torvalds's avatar
Linus Torvalds committed
769 770
	 */

771
	while (bio->bi_phys_segments > queue_max_segments(q)) {
Linus Torvalds's avatar
Linus Torvalds committed
772 773

		if (retried_segments)
774
			goto failed;
Linus Torvalds's avatar
Linus Torvalds committed
775 776 777 778 779 780

		retried_segments = 1;
		blk_recount_segments(q, bio);
	}

	/* If we may be able to merge these biovecs, force a recount */
781
	if (bio->bi_vcnt > 1 && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec)))
782
		bio_clear_flag(bio, BIO_SEG_VALID);
Linus Torvalds's avatar
Linus Torvalds committed
783

784
 done:
Linus Torvalds's avatar
Linus Torvalds committed
785
	return len;
786 787 788 789 790 791 792 793 794

 failed:
	bvec->bv_page = NULL;
	bvec->bv_len = 0;
	bvec->bv_offset = 0;
	bio->bi_vcnt--;
	bio->bi_iter.bi_size -= len;
	blk_recount_segments(q, bio);
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
795
}
796
EXPORT_SYMBOL(bio_add_pc_page);
797

Linus Torvalds's avatar
Linus Torvalds committed
798 799 800 801 802 803 804
/**
 *	bio_add_page	-	attempt to add page to bio
 *	@bio: destination bio
 *	@page: page to add
 *	@len: vec entry length
 *	@offset: vec entry offset
 *
805 806
 *	Attempt to add a page to the bio_vec maplist. This will only fail
 *	if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio.
Linus Torvalds's avatar
Linus Torvalds committed
807
 */
808 809
int bio_add_page(struct bio *bio, struct page *page,
		 unsigned int len, unsigned int offset)
Linus Torvalds's avatar
Linus Torvalds committed
810
{
811 812 813 814 815 816 817
	struct bio_vec *bv;

	/*
	 * cloned bio must not modify vec list
	 */
	if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
		return 0;
818

819 820 821 822 823 824 825
	/*
	 * For filesystems with a blocksize smaller than the pagesize
	 * we will often be called with the same page as last time and
	 * a consecutive offset.  Optimize this special case.
	 */
	if (bio->bi_vcnt > 0) {
		bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
826

827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845
		if (page == bv->bv_page &&
		    offset == bv->bv_offset + bv->bv_len) {
			bv->bv_len += len;
			goto done;
		}
	}

	if (bio->bi_vcnt >= bio->bi_max_vecs)
		return 0;

	bv		= &bio->bi_io_vec[bio->bi_vcnt];
	bv->bv_page	= page;
	bv->bv_len	= len;
	bv->bv_offset	= offset;

	bio->bi_vcnt++;
done:
	bio->bi_iter.bi_size += len;
	return len;
Linus Torvalds's avatar
Linus Torvalds committed
846
}
847
EXPORT_SYMBOL(bio_add_page);
Linus Torvalds's avatar
Linus Torvalds committed
848

849 850 851 852 853
struct submit_bio_ret {
	struct completion event;
	int error;
};

854
static void submit_bio_wait_endio(struct bio *bio)
855 856 857
{
	struct submit_bio_ret *ret = bio->bi_private;

858
	ret->error = bio->bi_error;
859 860 861 862 863 864 865 866 867 868
	complete(&ret->event);
}

/**
 * submit_bio_wait - submit a bio, and wait until it completes
 * @bio: The &struct bio which describes the I/O
 *
 * Simple wrapper around submit_bio(). Returns 0 on success, or the error from
 * bio_endio() on failure.
 */
869
int submit_bio_wait(struct bio *bio)
870 871 872 873 874 875
{
	struct submit_bio_ret ret;

	init_completion(&ret.event);
	bio->bi_private = &ret;
	bio->bi_end_io = submit_bio_wait_endio;
876 877
	bio->bi_rw |= REQ_SYNC;
	submit_bio(bio);
878
	wait_for_completion_io(&ret.event);
879 880 881 882 883

	return ret.error;
}
EXPORT_SYMBOL(submit_bio_wait);

Kent Overstreet's avatar
Kent Overstreet committed
884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899
/**
 * bio_advance - increment/complete a bio by some number of bytes
 * @bio:	bio to advance
 * @bytes:	number of bytes to complete
 *
 * This updates bi_sector, bi_size and bi_idx; if the number of bytes to
 * complete doesn't align with a bvec boundary, then bv_len and bv_offset will
 * be updated on the last bvec as well.
 *
 * @bio will then represent the remaining, uncompleted portion of the io.
 */
void bio_advance(struct bio *bio, unsigned bytes)
{
	if (bio_integrity(bio))
		bio_integrity_advance(bio, bytes);

Kent Overstreet's avatar
Kent Overstreet committed
900
	bio_advance_iter(bio, &bio->bi_iter, bytes);
Kent Overstreet's avatar
Kent Overstreet committed
901 902 903
}
EXPORT_SYMBOL(bio_advance);

Kent Overstreet's avatar
Kent Overstreet committed
904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931
/**
 * bio_alloc_pages - allocates a single page for each bvec in a bio
 * @bio: bio to allocate pages for
 * @gfp_mask: flags for allocation
 *
 * Allocates pages up to @bio->bi_vcnt.
 *
 * Returns 0 on success, -ENOMEM on failure. On failure, any allocated pages are
 * freed.
 */
int bio_alloc_pages(struct bio *bio, gfp_t gfp_mask)
{
	int i;
	struct bio_vec *bv;

	bio_for_each_segment_all(bv, bio, i) {
		bv->bv_page = alloc_page(gfp_mask);
		if (!bv->bv_page) {
			while (--bv >= bio->bi_io_vec)
				__free_page(bv->bv_page);
			return -ENOMEM;
		}
	}

	return 0;
}
EXPORT_SYMBOL(bio_alloc_pages);

Kent Overstreet's avatar
Kent Overstreet committed
932 933 934 935 936 937 938 939 940 941 942 943 944 945
/**
 * bio_copy_data - copy contents of data buffers from one chain of bios to
 * another
 * @src: source bio list
 * @dst: destination bio list
 *
 * If @src and @dst are single bios, bi_next must be NULL - otherwise, treats
 * @src and @dst as linked lists of bios.
 *
 * Stops when it reaches the end of either @src or @dst - that is, copies
 * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios).
 */
void bio_copy_data(struct bio *dst, struct bio *src)
{
946 947
	struct bvec_iter src_iter, dst_iter;
	struct bio_vec src_bv, dst_bv;
Kent Overstreet's avatar
Kent Overstreet committed
948
	void *src_p, *dst_p;
949
	unsigned bytes;
Kent Overstreet's avatar
Kent Overstreet committed
950

951 952
	src_iter = src->bi_iter;
	dst_iter = dst->bi_iter;
Kent Overstreet's avatar
Kent Overstreet committed
953 954

	while (1) {
955 956 957 958
		if (!src_iter.bi_size) {
			src = src->bi_next;
			if (!src)
				break;
Kent Overstreet's avatar
Kent Overstreet committed
959

960
			src_iter = src->bi_iter;
Kent Overstreet's avatar
Kent Overstreet committed
961 962
		}

963 964 965 966
		if (!dst_iter.bi_size) {
			dst = dst->bi_next;
			if (!dst)
				break;
Kent Overstreet's avatar
Kent Overstreet committed
967

968
			dst_iter = dst->bi_iter;
Kent Overstreet's avatar
Kent Overstreet committed
969 970
		}

971 972 973 974
		src_bv = bio_iter_iovec(src, src_iter);
		dst_bv = bio_iter_iovec(dst, dst_iter);

		bytes = min(src_bv.bv_len, dst_bv.bv_len);
Kent Overstreet's avatar
Kent Overstreet committed
975

976 977
		src_p = kmap_atomic(src_bv.bv_page);
		dst_p = kmap_atomic(dst_bv.bv_page);
Kent Overstreet's avatar
Kent Overstreet committed
978

979 980
		memcpy(dst_p + dst_bv.bv_offset,
		       src_p + src_bv.bv_offset,
Kent Overstreet's avatar
Kent Overstreet committed
981 982 983 984 985
		       bytes);

		kunmap_atomic(dst_p);
		kunmap_atomic(src_p);

986 987
		bio_advance_iter(src, &src_iter, bytes);
		bio_advance_iter(dst, &dst_iter, bytes);
Kent Overstreet's avatar
Kent Overstreet committed
988 989 990 991
	}
}
EXPORT_SYMBOL(bio_copy_data);

Linus Torvalds's avatar
Linus Torvalds committed
992
struct bio_map_data {
993
	int is_our_pages;
994 995
	struct iov_iter iter;
	struct iovec iov[];
Linus Torvalds's avatar
Linus Torvalds committed
996 997
};

998
static struct bio_map_data *bio_alloc_map_data(unsigned int iov_count,
999
					       gfp_t gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
1000
{
1001 1002
	if (iov_count > UIO_MAXIOV)
		return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
1003

1004
	return kmalloc(sizeof(struct bio_map_data) +
1005
		       sizeof(struct iovec) * iov_count, gfp_mask);
Linus Torvalds's avatar
Linus Torvalds committed
1006 1007
}

1008 1009 1010 1011 1012 1013 1014 1015 1016
/**
 * bio_copy_from_iter - copy all pages from iov_iter to bio
 * @bio: The &struct bio which describes the I/O as destination
 * @iter: iov_iter as source
 *
 * Copy all pages from iov_iter to bio.
 * Returns 0 on success, or error on failure.
 */
static int bio_copy_from_iter(struct bio *bio, struct iov_iter iter)
1017
{
1018
	int i;
1019 1020
	struct bio_vec *bvec;

1021
	bio_for_each_segment_all(bvec, bio, i) {
1022
		ssize_t ret;
1023

1024 1025 1026 1027 1028 1029 1030 1031 1032 1033
		ret = copy_page_from_iter(bvec->bv_page,
					  bvec->bv_offset,
					  bvec->bv_len,
					  &iter);

		if (!iov_iter_count(&iter))
			break;

		if (ret < bvec->bv_len)
			return -EFAULT;
1034 1035
	}

1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067
	return 0;
}

/**
 * bio_copy_to_iter - copy all pages from bio to iov_iter
 * @bio: The &struct bio which describes the I/O as source
 * @iter: iov_iter as destination
 *
 * Copy all pages from bio to iov_iter.
 * Returns 0 on success, or error on failure.
 */
static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter)
{
	int i;
	struct bio_vec *bvec;

	bio_for_each_segment_all(bvec, bio, i) {
		ssize_t ret;

		ret = copy_page_to_iter(bvec->bv_page,
					bvec->bv_offset,
					bvec->bv_len,
					&iter);

		if (!iov_iter_count(&iter))
			break;

		if (ret < bvec->bv_len)
			return -EFAULT;
	}

	return 0;
1068 1069
}

1070 1071 1072 1073 1074 1075 1076 1077 1078
static void bio_free_pages(struct bio *bio)
{
	struct bio_vec *bvec;
	int i;

	bio_for_each_segment_all(bvec, bio, i)
		__free_page(bvec->bv_page);
}

Linus Torvalds's avatar
Linus Torvalds committed
1079 1080 1081 1082
/**
 *	bio_uncopy_user	-	finish previously mapped bio
 *	@bio: bio being terminated
 *
1083
 *	Free pages allocated from bio_copy_user_iov() and write back data
Linus Torvalds's avatar
Linus Torvalds committed
1084 1085 1086 1087 1088
 *	to user space in case of a read.
 */
int bio_uncopy_user(struct bio *bio)
{
	struct bio_map_data *bmd = bio->bi_private;
1089
	int ret = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1090

1091 1092 1093
	if (!bio_flagged(bio, BIO_NULL_MAPPED)) {
		/*
		 * if we're in a workqueue, the request is orphaned, so
1094 1095
		 * don't copy into a random user address space, just free
		 * and return -EINTR so user space doesn't expect any data.
1096
		 */
1097 1098 1099
		if (!current->mm)
			ret = -EINTR;
		else if (bio_data_dir(bio) == READ)
1100
			ret = bio_copy_to_iter(bio, bmd->iter);
1101 1102
		if (bmd->is_our_pages)
			bio_free_pages(bio);
1103
	}
1104
	kfree(bmd);
Linus Torvalds's avatar
Linus Torvalds committed
1105 1106 1107 1108 1109
	bio_put(bio);
	return ret;
}

/**
1110
 *	bio_copy_user_iov	-	copy user data to bio
1111 1112 1113 1114
 *	@q:		destination block queue
 *	@map_data:	pointer to the rq_map_data holding pages (if necessary)
 *	@iter:		iovec iterator
 *	@gfp_mask:	memory allocation flags
Linus Torvalds's avatar
Linus Torvalds committed
1115 1116 1117 1118 1119
 *
 *	Prepares and returns a bio for indirect user io, bouncing data
 *	to/from kernel pages as necessary. Must be paired with
 *	call bio_uncopy_user() on io completion.
 */
1120 1121
struct bio *bio_copy_user_iov(struct request_queue *q,
			      struct rq_map_data *map_data,
1122 1123
			      const struct iov_iter *iter,
			      gfp_t gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
1124 1125 1126 1127 1128
{
	struct bio_map_data *bmd;
	struct page *page;
	struct bio *bio;
	int i, ret;
1129
	int nr_pages = 0;
1130
	unsigned int len = iter->count;
Geliang Tang's avatar
Geliang Tang committed
1131
	unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0;
Linus Torvalds's avatar
Linus Torvalds committed
1132

1133
	for (i = 0; i < iter->nr_segs; i++) {
1134 1135 1136 1137
		unsigned long uaddr;
		unsigned long end;
		unsigned long start;

1138 1139 1140
		uaddr = (unsigned long) iter->iov[i].iov_base;
		end = (uaddr + iter->iov[i].iov_len + PAGE_SIZE - 1)
			>> PAGE_SHIFT;
1141 1142
		start = uaddr >> PAGE_SHIFT;

1143 1144 1145 1146 1147 1148
		/*
		 * Overflow, abort
		 */
		if (end < start)
			return ERR_PTR(-EINVAL);

1149 1150 1151
		nr_pages += end - start;
	}

1152 1153 1154
	if (offset)
		nr_pages++;

1155
	bmd = bio_alloc_map_data(iter->nr_segs, gfp_mask);
Linus Torvalds's avatar
Linus Torvalds committed
1156 1157 1158
	if (!bmd)
		return ERR_PTR(-ENOMEM);

1159 1160 1161 1162 1163 1164 1165 1166 1167 1168
	/*
	 * We need to do a deep copy of the iov_iter including the iovecs.
	 * The caller provided iov might point to an on-stack or otherwise
	 * shortlived one.
	 */
	bmd->is_our_pages = map_data ? 0 : 1;
	memcpy(bmd->iov, iter->iov, sizeof(struct iovec) * iter->nr_segs);
	iov_iter_init(&bmd->iter, iter->type, bmd->iov,
			iter->nr_segs, iter->count);

Linus Torvalds's avatar
Linus Torvalds committed
1169
	ret = -ENOMEM;
1170
	bio = bio_kmalloc(gfp_mask, nr_pages);
Linus Torvalds's avatar
Linus Torvalds committed
1171 1172 1173
	if (!bio)
		goto out_bmd;

1174
	if (iter->type & WRITE)
1175
		bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
Linus Torvalds's avatar
Linus Torvalds committed
1176 1177

	ret = 0;
1178 1179

	if (map_data) {
1180
		nr_pages = 1 << map_data->page_order;
1181 1182
		i = map_data->offset / PAGE_SIZE;
	}
Linus Torvalds's avatar
Linus Torvalds committed
1183
	while (len) {
1184
		unsigned int bytes = PAGE_SIZE;
Linus Torvalds's avatar
Linus Torvalds committed
1185

1186 1187
		bytes -= offset;

Linus Torvalds's avatar
Linus Torvalds committed
1188 1189 1190
		if (bytes > len)
			bytes = len;

1191
		if (map_data) {
1192
			if (i == map_data->nr_entries * nr_pages) {
1193 1194 1195
				ret = -ENOMEM;
				break;
			}
1196 1197 1198 1199 1200 1201

			page = map_data->pages[i / nr_pages];
			page += (i % nr_pages);

			i++;
		} else {
1202
			page = alloc_page(q->bounce_gfp | gfp_mask);
1203 1204 1205 1206
			if (!page) {
				ret = -ENOMEM;
				break;
			}
Linus Torvalds's avatar
Linus Torvalds committed
1207 1208
		}

1209
		if (bio_add_pc_page(q, bio, page, bytes, offset) < bytes)
Linus Torvalds's avatar
Linus Torvalds committed
1210 1211 1212
			break;

		len -= bytes;
1213
		offset = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1214 1215 1216 1217 1218 1219 1220 1221
	}

	if (ret)
		goto cleanup;

	/*
	 * success
	 */
1222
	if (((iter->type & WRITE) && (!map_data || !map_data->null_mapped)) ||
1223
	    (map_data && map_data->from_user)) {
1224
		ret = bio_copy_from_iter(bio, *iter);
FUJITA Tomonori's avatar