bio.c 49.5 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
/*
2
 * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
Linus Torvalds's avatar
Linus Torvalds committed
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public Licens
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-
 *
 */
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
22
#include <linux/uio.h>
23
#include <linux/iocontext.h>
Linus Torvalds's avatar
Linus Torvalds committed
24 25 26
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/kernel.h>
27
#include <linux/export.h>
Linus Torvalds's avatar
Linus Torvalds committed
28 29
#include <linux/mempool.h>
#include <linux/workqueue.h>
30
#include <linux/cgroup.h>
Linus Torvalds's avatar
Linus Torvalds committed
31

32
#include <trace/events/block.h>
33
#include "blk.h"
34

35 36 37 38 39 40
/*
 * Test patch to inline a certain number of bi_io_vec's inside the bio
 * itself, to shrink a bio data allocation from two mempool calls to one
 */
#define BIO_INLINE_VECS		4

Linus Torvalds's avatar
Linus Torvalds committed
41 42 43 44 45 46
/*
 * if you change this list, also change bvec_alloc or things will
 * break badly! cannot be bigger than what you can fit into an
 * unsigned short
 */
#define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) }
47
static struct biovec_slab bvec_slabs[BVEC_POOL_NR] __read_mostly = {
Linus Torvalds's avatar
Linus Torvalds committed
48 49 50 51 52 53 54 55
	BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES),
};
#undef BV

/*
 * fs_bio_set is the bio_set containing bio and iovec memory pools used by
 * IO code that does not need private memory pools.
 */
56
struct bio_set *fs_bio_set;
57
EXPORT_SYMBOL(fs_bio_set);
Linus Torvalds's avatar
Linus Torvalds committed
58

59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
/*
 * Our slab pool management
 */
struct bio_slab {
	struct kmem_cache *slab;
	unsigned int slab_ref;
	unsigned int slab_size;
	char name[8];
};
static DEFINE_MUTEX(bio_slab_lock);
static struct bio_slab *bio_slabs;
static unsigned int bio_slab_nr, bio_slab_max;

static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size)
{
	unsigned int sz = sizeof(struct bio) + extra_size;
	struct kmem_cache *slab = NULL;
76
	struct bio_slab *bslab, *new_bio_slabs;
77
	unsigned int new_bio_slab_max;
78 79 80 81 82 83
	unsigned int i, entry = -1;

	mutex_lock(&bio_slab_lock);

	i = 0;
	while (i < bio_slab_nr) {
84
		bslab = &bio_slabs[i];
85 86 87 88 89 90 91 92 93 94 95 96 97 98 99

		if (!bslab->slab && entry == -1)
			entry = i;
		else if (bslab->slab_size == sz) {
			slab = bslab->slab;
			bslab->slab_ref++;
			break;
		}
		i++;
	}

	if (slab)
		goto out_unlock;

	if (bio_slab_nr == bio_slab_max && entry == -1) {
100
		new_bio_slab_max = bio_slab_max << 1;
101
		new_bio_slabs = krealloc(bio_slabs,
102
					 new_bio_slab_max * sizeof(struct bio_slab),
103 104
					 GFP_KERNEL);
		if (!new_bio_slabs)
105
			goto out_unlock;
106
		bio_slab_max = new_bio_slab_max;
107
		bio_slabs = new_bio_slabs;
108 109 110 111 112 113 114
	}
	if (entry == -1)
		entry = bio_slab_nr++;

	bslab = &bio_slabs[entry];

	snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry);
115 116
	slab = kmem_cache_create(bslab->name, sz, ARCH_KMALLOC_MINALIGN,
				 SLAB_HWCACHE_ALIGN, NULL);
117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
	if (!slab)
		goto out_unlock;

	bslab->slab = slab;
	bslab->slab_ref = 1;
	bslab->slab_size = sz;
out_unlock:
	mutex_unlock(&bio_slab_lock);
	return slab;
}

static void bio_put_slab(struct bio_set *bs)
{
	struct bio_slab *bslab = NULL;
	unsigned int i;

	mutex_lock(&bio_slab_lock);

	for (i = 0; i < bio_slab_nr; i++) {
		if (bs->bio_slab == bio_slabs[i].slab) {
			bslab = &bio_slabs[i];
			break;
		}
	}

	if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))
		goto out;

	WARN_ON(!bslab->slab_ref);

	if (--bslab->slab_ref)
		goto out;

	kmem_cache_destroy(bslab->slab);
	bslab->slab = NULL;

out:
	mutex_unlock(&bio_slab_lock);
}

157 158 159 160 161
unsigned int bvec_nr_vecs(unsigned short idx)
{
	return bvec_slabs[idx].nr_vecs;
}

162
void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
163
{
164 165 166 167 168
	if (!idx)
		return;
	idx--;

	BIO_BUG_ON(idx >= BVEC_POOL_NR);
169

170
	if (idx == BVEC_POOL_MAX) {
171
		mempool_free(bv, pool);
172
	} else {
173 174 175 176 177 178
		struct biovec_slab *bvs = bvec_slabs + idx;

		kmem_cache_free(bvs->slab, bv);
	}
}

179 180
struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
			   mempool_t *pool)
Linus Torvalds's avatar
Linus Torvalds committed
181 182 183
{
	struct bio_vec *bvl;

184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213
	/*
	 * see comment near bvec_array define!
	 */
	switch (nr) {
	case 1:
		*idx = 0;
		break;
	case 2 ... 4:
		*idx = 1;
		break;
	case 5 ... 16:
		*idx = 2;
		break;
	case 17 ... 64:
		*idx = 3;
		break;
	case 65 ... 128:
		*idx = 4;
		break;
	case 129 ... BIO_MAX_PAGES:
		*idx = 5;
		break;
	default:
		return NULL;
	}

	/*
	 * idx now points to the pool we want to allocate from. only the
	 * 1-vec entry pool is mempool backed.
	 */
214
	if (*idx == BVEC_POOL_MAX) {
215
fallback:
216
		bvl = mempool_alloc(pool, gfp_mask);
217 218
	} else {
		struct biovec_slab *bvs = bvec_slabs + *idx;
219
		gfp_t __gfp_mask = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_IO);
220

Jens Axboe's avatar
Jens Axboe committed
221
		/*
222 223 224
		 * Make this allocation restricted and don't dump info on
		 * allocation failures, since we'll fallback to the mempool
		 * in case of failure.
Jens Axboe's avatar
Jens Axboe committed
225
		 */
226
		__gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
Linus Torvalds's avatar
Linus Torvalds committed
227

Jens Axboe's avatar
Jens Axboe committed
228
		/*
229
		 * Try a slab allocation. If this fails and __GFP_DIRECT_RECLAIM
230
		 * is set, retry with the 1-entry mempool
Jens Axboe's avatar
Jens Axboe committed
231
		 */
232
		bvl = kmem_cache_alloc(bvs->slab, __gfp_mask);
233
		if (unlikely(!bvl && (gfp_mask & __GFP_DIRECT_RECLAIM))) {
234
			*idx = BVEC_POOL_MAX;
235 236 237 238
			goto fallback;
		}
	}

239
	(*idx)++;
Linus Torvalds's avatar
Linus Torvalds committed
240 241 242
	return bvl;
}

243
void bio_uninit(struct bio *bio)
Linus Torvalds's avatar
Linus Torvalds committed
244
{
Kent Overstreet's avatar
Kent Overstreet committed
245 246
	bio_disassociate_task(bio);
}
247
EXPORT_SYMBOL(bio_uninit);
248

Kent Overstreet's avatar
Kent Overstreet committed
249 250 251 252 253
static void bio_free(struct bio *bio)
{
	struct bio_set *bs = bio->bi_pool;
	void *p;

254
	bio_uninit(bio);
Kent Overstreet's avatar
Kent Overstreet committed
255 256

	if (bs) {
257
		bvec_free(bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));
Kent Overstreet's avatar
Kent Overstreet committed
258 259 260 261 262

		/*
		 * If we have front padding, adjust the bio pointer before freeing
		 */
		p = bio;
263 264
		p -= bs->front_pad;

Kent Overstreet's avatar
Kent Overstreet committed
265 266 267 268 269
		mempool_free(p, bs->bio_pool);
	} else {
		/* Bio was allocated by bio_kmalloc() */
		kfree(bio);
	}
Peter Osterlund's avatar
Peter Osterlund committed
270 271
}

272 273 274 275 276
/*
 * Users of this function have their own bio allocation. Subsequently,
 * they must remember to pair any call to bio_init() with bio_uninit()
 * when IO has completed, or when the bio is released.
 */
277 278
void bio_init(struct bio *bio, struct bio_vec *table,
	      unsigned short max_vecs)
Linus Torvalds's avatar
Linus Torvalds committed
279
{
Jens Axboe's avatar
Jens Axboe committed
280
	memset(bio, 0, sizeof(*bio));
281
	atomic_set(&bio->__bi_remaining, 1);
282
	atomic_set(&bio->__bi_cnt, 1);
283 284 285

	bio->bi_io_vec = table;
	bio->bi_max_vecs = max_vecs;
Linus Torvalds's avatar
Linus Torvalds committed
286
}
287
EXPORT_SYMBOL(bio_init);
Linus Torvalds's avatar
Linus Torvalds committed
288

Kent Overstreet's avatar
Kent Overstreet committed
289 290 291 292 293 294 295 296 297 298 299 300 301 302
/**
 * bio_reset - reinitialize a bio
 * @bio:	bio to reset
 *
 * Description:
 *   After calling bio_reset(), @bio will be in the same state as a freshly
 *   allocated bio returned bio bio_alloc_bioset() - the only fields that are
 *   preserved are the ones that are initialized by bio_alloc_bioset(). See
 *   comment in struct bio.
 */
void bio_reset(struct bio *bio)
{
	unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);

303
	bio_uninit(bio);
Kent Overstreet's avatar
Kent Overstreet committed
304 305

	memset(bio, 0, BIO_RESET_BYTES);
306
	bio->bi_flags = flags;
307
	atomic_set(&bio->__bi_remaining, 1);
Kent Overstreet's avatar
Kent Overstreet committed
308 309 310
}
EXPORT_SYMBOL(bio_reset);

311
static struct bio *__bio_chain_endio(struct bio *bio)
Kent Overstreet's avatar
Kent Overstreet committed
312
{
313 314
	struct bio *parent = bio->bi_private;

315 316
	if (!parent->bi_status)
		parent->bi_status = bio->bi_status;
Kent Overstreet's avatar
Kent Overstreet committed
317
	bio_put(bio);
318 319 320 321 322 323
	return parent;
}

static void bio_chain_endio(struct bio *bio)
{
	bio_endio(__bio_chain_endio(bio));
Kent Overstreet's avatar
Kent Overstreet committed
324 325 326 327
}

/**
 * bio_chain - chain bio completions
328 329
 * @bio: the target bio
 * @parent: the @bio's parent bio
Kent Overstreet's avatar
Kent Overstreet committed
330 331 332 333 334 335 336 337 338 339 340 341 342
 *
 * The caller won't have a bi_end_io called when @bio completes - instead,
 * @parent's bi_end_io won't be called until both @parent and @bio have
 * completed; the chained bio will also be freed when it completes.
 *
 * The caller must not set bi_private or bi_end_io in @bio.
 */
void bio_chain(struct bio *bio, struct bio *parent)
{
	BUG_ON(bio->bi_private || bio->bi_end_io);

	bio->bi_private = parent;
	bio->bi_end_io	= bio_chain_endio;
343
	bio_inc_remaining(parent);
Kent Overstreet's avatar
Kent Overstreet committed
344 345 346
}
EXPORT_SYMBOL(bio_chain);

347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368
static void bio_alloc_rescue(struct work_struct *work)
{
	struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
	struct bio *bio;

	while (1) {
		spin_lock(&bs->rescue_lock);
		bio = bio_list_pop(&bs->rescue_list);
		spin_unlock(&bs->rescue_lock);

		if (!bio)
			break;

		generic_make_request(bio);
	}
}

static void punt_bios_to_rescuer(struct bio_set *bs)
{
	struct bio_list punt, nopunt;
	struct bio *bio;

369 370
	if (WARN_ON_ONCE(!bs->rescue_workqueue))
		return;
371 372 373 374 375 376 377 378 379 380 381 382 383 384
	/*
	 * In order to guarantee forward progress we must punt only bios that
	 * were allocated from this bio_set; otherwise, if there was a bio on
	 * there for a stacking driver higher up in the stack, processing it
	 * could require allocating bios from this bio_set, and doing that from
	 * our own rescuer would be bad.
	 *
	 * Since bio lists are singly linked, pop them all instead of trying to
	 * remove from the middle of the list:
	 */

	bio_list_init(&punt);
	bio_list_init(&nopunt);

385
	while ((bio = bio_list_pop(&current->bio_list[0])))
386
		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
387
	current->bio_list[0] = nopunt;
388

389 390 391 392
	bio_list_init(&nopunt);
	while ((bio = bio_list_pop(&current->bio_list[1])))
		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
	current->bio_list[1] = nopunt;
393 394 395 396 397 398 399 400

	spin_lock(&bs->rescue_lock);
	bio_list_merge(&bs->rescue_list, &punt);
	spin_unlock(&bs->rescue_lock);

	queue_work(bs->rescue_workqueue, &bs->rescue_work);
}

Linus Torvalds's avatar
Linus Torvalds committed
401 402
/**
 * bio_alloc_bioset - allocate a bio for I/O
403
 * @gfp_mask:   the GFP_* mask given to the slab allocator
Linus Torvalds's avatar
Linus Torvalds committed
404
 * @nr_iovecs:	number of iovecs to pre-allocate
405
 * @bs:		the bio_set to allocate from.
Linus Torvalds's avatar
Linus Torvalds committed
406 407
 *
 * Description:
408 409 410
 *   If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is
 *   backed by the @bs's mempool.
 *
411 412 413 414 415 416
 *   When @bs is not NULL, if %__GFP_DIRECT_RECLAIM is set then bio_alloc will
 *   always be able to allocate a bio. This is due to the mempool guarantees.
 *   To make this work, callers must never allocate more than 1 bio at a time
 *   from this pool. Callers that need to allocate more than 1 bio must always
 *   submit the previously allocated bio for IO before attempting to allocate
 *   a new one. Failure to do so can cause deadlocks under memory pressure.
417
 *
418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
 *   Note that when running under generic_make_request() (i.e. any block
 *   driver), bios are not submitted until after you return - see the code in
 *   generic_make_request() that converts recursion into iteration, to prevent
 *   stack overflows.
 *
 *   This would normally mean allocating multiple bios under
 *   generic_make_request() would be susceptible to deadlocks, but we have
 *   deadlock avoidance code that resubmits any blocked bios from a rescuer
 *   thread.
 *
 *   However, we do not guarantee forward progress for allocations from other
 *   mempools. Doing multiple allocations from the same mempool under
 *   generic_make_request() should be avoided - instead, use bio_set's front_pad
 *   for per bio allocations.
 *
433 434 435
 *   RETURNS:
 *   Pointer to new bio on success, NULL on failure.
 */
436 437
struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
			     struct bio_set *bs)
Linus Torvalds's avatar
Linus Torvalds committed
438
{
439
	gfp_t saved_gfp = gfp_mask;
440 441
	unsigned front_pad;
	unsigned inline_vecs;
442
	struct bio_vec *bvl = NULL;
Tejun Heo's avatar
Tejun Heo committed
443 444 445
	struct bio *bio;
	void *p;

446 447 448 449 450 451 452 453 454 455
	if (!bs) {
		if (nr_iovecs > UIO_MAXIOV)
			return NULL;

		p = kmalloc(sizeof(struct bio) +
			    nr_iovecs * sizeof(struct bio_vec),
			    gfp_mask);
		front_pad = 0;
		inline_vecs = nr_iovecs;
	} else {
456 457 458
		/* should not use nobvec bioset for nr_iovecs > 0 */
		if (WARN_ON_ONCE(!bs->bvec_pool && nr_iovecs > 0))
			return NULL;
459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474
		/*
		 * generic_make_request() converts recursion to iteration; this
		 * means if we're running beneath it, any bios we allocate and
		 * submit will not be submitted (and thus freed) until after we
		 * return.
		 *
		 * This exposes us to a potential deadlock if we allocate
		 * multiple bios from the same bio_set() while running
		 * underneath generic_make_request(). If we were to allocate
		 * multiple bios (say a stacking block driver that was splitting
		 * bios), we would deadlock if we exhausted the mempool's
		 * reserve.
		 *
		 * We solve this, and guarantee forward progress, with a rescuer
		 * workqueue per bio_set. If we go to allocate and there are
		 * bios on current->bio_list, we first try the allocation
475 476 477
		 * without __GFP_DIRECT_RECLAIM; if that fails, we punt those
		 * bios we would be blocking to the rescuer workqueue before
		 * we retry with the original gfp_flags.
478 479
		 */

480 481
		if (current->bio_list &&
		    (!bio_list_empty(&current->bio_list[0]) ||
482 483
		     !bio_list_empty(&current->bio_list[1])) &&
		    bs->rescue_workqueue)
484
			gfp_mask &= ~__GFP_DIRECT_RECLAIM;
485

486
		p = mempool_alloc(bs->bio_pool, gfp_mask);
487 488 489 490 491 492
		if (!p && gfp_mask != saved_gfp) {
			punt_bios_to_rescuer(bs);
			gfp_mask = saved_gfp;
			p = mempool_alloc(bs->bio_pool, gfp_mask);
		}

493 494 495 496
		front_pad = bs->front_pad;
		inline_vecs = BIO_INLINE_VECS;
	}

Tejun Heo's avatar
Tejun Heo committed
497 498
	if (unlikely(!p))
		return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
499

500
	bio = p + front_pad;
501
	bio_init(bio, NULL, 0);
502

503
	if (nr_iovecs > inline_vecs) {
504 505
		unsigned long idx = 0;

506
		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
507 508 509
		if (!bvl && gfp_mask != saved_gfp) {
			punt_bios_to_rescuer(bs);
			gfp_mask = saved_gfp;
510
			bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
511 512
		}

513 514
		if (unlikely(!bvl))
			goto err_free;
515

516
		bio->bi_flags |= idx << BVEC_POOL_OFFSET;
517 518
	} else if (nr_iovecs) {
		bvl = bio->bi_inline_vecs;
Linus Torvalds's avatar
Linus Torvalds committed
519
	}
520 521

	bio->bi_pool = bs;
522 523
	bio->bi_max_vecs = nr_iovecs;
	bio->bi_io_vec = bvl;
Linus Torvalds's avatar
Linus Torvalds committed
524
	return bio;
525 526

err_free:
Tejun Heo's avatar
Tejun Heo committed
527
	mempool_free(p, bs->bio_pool);
528
	return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
529
}
530
EXPORT_SYMBOL(bio_alloc_bioset);
Linus Torvalds's avatar
Linus Torvalds committed
531 532 533 534

void zero_fill_bio(struct bio *bio)
{
	unsigned long flags;
535 536
	struct bio_vec bv;
	struct bvec_iter iter;
Linus Torvalds's avatar
Linus Torvalds committed
537

538 539 540 541
	bio_for_each_segment(bv, bio, iter) {
		char *data = bvec_kmap_irq(&bv, &flags);
		memset(data, 0, bv.bv_len);
		flush_dcache_page(bv.bv_page);
Linus Torvalds's avatar
Linus Torvalds committed
542 543 544 545 546 547 548 549 550 551 552
		bvec_kunmap_irq(data, &flags);
	}
}
EXPORT_SYMBOL(zero_fill_bio);

/**
 * bio_put - release a reference to a bio
 * @bio:   bio to release reference to
 *
 * Description:
 *   Put a reference to a &struct bio, either one you have gotten with
553
 *   bio_alloc, bio_get or bio_clone_*. The last put of a bio will free it.
Linus Torvalds's avatar
Linus Torvalds committed
554 555 556
 **/
void bio_put(struct bio *bio)
{
557
	if (!bio_flagged(bio, BIO_REFFED))
Kent Overstreet's avatar
Kent Overstreet committed
558
		bio_free(bio);
559 560 561 562 563 564 565 566 567
	else {
		BIO_BUG_ON(!atomic_read(&bio->__bi_cnt));

		/*
		 * last put frees it
		 */
		if (atomic_dec_and_test(&bio->__bi_cnt))
			bio_free(bio);
	}
Linus Torvalds's avatar
Linus Torvalds committed
568
}
569
EXPORT_SYMBOL(bio_put);
Linus Torvalds's avatar
Linus Torvalds committed
570

571
inline int bio_phys_segments(struct request_queue *q, struct bio *bio)
Linus Torvalds's avatar
Linus Torvalds committed
572 573 574 575 576 577
{
	if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
		blk_recount_segments(q, bio);

	return bio->bi_phys_segments;
}
578
EXPORT_SYMBOL(bio_phys_segments);
Linus Torvalds's avatar
Linus Torvalds committed
579

Kent Overstreet's avatar
Kent Overstreet committed
580 581 582 583 584 585 586 587 588 589 590 591 592
/**
 * 	__bio_clone_fast - clone a bio that shares the original bio's biovec
 * 	@bio: destination bio
 * 	@bio_src: bio to clone
 *
 *	Clone a &bio. Caller will own the returned bio, but not
 *	the actual data it points to. Reference count of returned
 * 	bio will be one.
 *
 * 	Caller must ensure that @bio_src is not freed before @bio.
 */
void __bio_clone_fast(struct bio *bio, struct bio *bio_src)
{
593
	BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio));
Kent Overstreet's avatar
Kent Overstreet committed
594 595

	/*
596
	 * most users will be overriding ->bi_disk with a new target,
Kent Overstreet's avatar
Kent Overstreet committed
597 598
	 * so we don't set nor calculate new physical/hw segment counts here
	 */
599
	bio->bi_disk = bio_src->bi_disk;
600
	bio->bi_partno = bio_src->bi_partno;
601
	bio_set_flag(bio, BIO_CLONED);
Jens Axboe's avatar
Jens Axboe committed
602
	bio->bi_opf = bio_src->bi_opf;
603
	bio->bi_write_hint = bio_src->bi_write_hint;
Kent Overstreet's avatar
Kent Overstreet committed
604 605
	bio->bi_iter = bio_src->bi_iter;
	bio->bi_io_vec = bio_src->bi_io_vec;
606 607

	bio_clone_blkcg_association(bio, bio_src);
Kent Overstreet's avatar
Kent Overstreet committed
608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643
}
EXPORT_SYMBOL(__bio_clone_fast);

/**
 *	bio_clone_fast - clone a bio that shares the original bio's biovec
 *	@bio: bio to clone
 *	@gfp_mask: allocation priority
 *	@bs: bio_set to allocate from
 *
 * 	Like __bio_clone_fast, only also allocates the returned bio
 */
struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)
{
	struct bio *b;

	b = bio_alloc_bioset(gfp_mask, 0, bs);
	if (!b)
		return NULL;

	__bio_clone_fast(b, bio);

	if (bio_integrity(bio)) {
		int ret;

		ret = bio_integrity_clone(b, bio, gfp_mask);

		if (ret < 0) {
			bio_put(b);
			return NULL;
		}
	}

	return b;
}
EXPORT_SYMBOL(bio_clone_fast);

644 645 646 647 648 649 650 651 652 653 654
/**
 * 	bio_clone_bioset - clone a bio
 * 	@bio_src: bio to clone
 *	@gfp_mask: allocation priority
 *	@bs: bio_set to allocate from
 *
 *	Clone bio. Caller will own the returned bio, but not the actual data it
 *	points to. Reference count of returned bio will be one.
 */
struct bio *bio_clone_bioset(struct bio *bio_src, gfp_t gfp_mask,
			     struct bio_set *bs)
Linus Torvalds's avatar
Linus Torvalds committed
655
{
656 657 658
	struct bvec_iter iter;
	struct bio_vec bv;
	struct bio *bio;
Linus Torvalds's avatar
Linus Torvalds committed
659

660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681
	/*
	 * Pre immutable biovecs, __bio_clone() used to just do a memcpy from
	 * bio_src->bi_io_vec to bio->bi_io_vec.
	 *
	 * We can't do that anymore, because:
	 *
	 *  - The point of cloning the biovec is to produce a bio with a biovec
	 *    the caller can modify: bi_idx and bi_bvec_done should be 0.
	 *
	 *  - The original bio could've had more than BIO_MAX_PAGES biovecs; if
	 *    we tried to clone the whole thing bio_alloc_bioset() would fail.
	 *    But the clone should succeed as long as the number of biovecs we
	 *    actually need to allocate is fewer than BIO_MAX_PAGES.
	 *
	 *  - Lastly, bi_vcnt should not be looked at or relied upon by code
	 *    that does not own the bio - reason being drivers don't use it for
	 *    iterating over the biovec anymore, so expecting it to be kept up
	 *    to date (i.e. for clones that share the parent biovec) is just
	 *    asking for trouble and would force extra work on
	 *    __bio_clone_fast() anyways.
	 */

682
	bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs);
683
	if (!bio)
684
		return NULL;
685
	bio->bi_disk		= bio_src->bi_disk;
Jens Axboe's avatar
Jens Axboe committed
686
	bio->bi_opf		= bio_src->bi_opf;
687
	bio->bi_write_hint	= bio_src->bi_write_hint;
688 689
	bio->bi_iter.bi_sector	= bio_src->bi_iter.bi_sector;
	bio->bi_iter.bi_size	= bio_src->bi_iter.bi_size;
690

Adrian Hunter's avatar
Adrian Hunter committed
691 692 693
	switch (bio_op(bio)) {
	case REQ_OP_DISCARD:
	case REQ_OP_SECURE_ERASE:
694
	case REQ_OP_WRITE_ZEROES:
Adrian Hunter's avatar
Adrian Hunter committed
695 696
		break;
	case REQ_OP_WRITE_SAME:
697
		bio->bi_io_vec[bio->bi_vcnt++] = bio_src->bi_io_vec[0];
Adrian Hunter's avatar
Adrian Hunter committed
698 699
		break;
	default:
700
		bio_for_each_segment(bv, bio_src, iter)
Adrian Hunter's avatar
Adrian Hunter committed
701 702
			bio->bi_io_vec[bio->bi_vcnt++] = bv;
		break;
703 704
	}

705 706
	if (bio_integrity(bio_src)) {
		int ret;
707

708
		ret = bio_integrity_clone(bio, bio_src, gfp_mask);
709
		if (ret < 0) {
710
			bio_put(bio);
711
			return NULL;
712
		}
Peter Osterlund's avatar
Peter Osterlund committed
713
	}
Linus Torvalds's avatar
Linus Torvalds committed
714

715 716
	bio_clone_blkcg_association(bio, bio_src);

717
	return bio;
Linus Torvalds's avatar
Linus Torvalds committed
718
}
719
EXPORT_SYMBOL(bio_clone_bioset);
Linus Torvalds's avatar
Linus Torvalds committed
720 721

/**
722 723 724 725 726 727
 *	bio_add_pc_page	-	attempt to add page to bio
 *	@q: the target queue
 *	@bio: destination bio
 *	@page: page to add
 *	@len: vec entry length
 *	@offset: vec entry offset
Linus Torvalds's avatar
Linus Torvalds committed
728
 *
729 730 731 732 733 734
 *	Attempt to add a page to the bio_vec maplist. This can fail for a
 *	number of reasons, such as the bio being full or target block device
 *	limitations. The target block device must allow bio's up to PAGE_SIZE,
 *	so it is always possible to add a single page to an empty bio.
 *
 *	This should only be used by REQ_PC bios.
Linus Torvalds's avatar
Linus Torvalds committed
735
 */
736 737
int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page
		    *page, unsigned int len, unsigned int offset)
Linus Torvalds's avatar
Linus Torvalds committed
738 739 740 741 742 743 744 745 746 747
{
	int retried_segments = 0;
	struct bio_vec *bvec;

	/*
	 * cloned bio must not modify vec list
	 */
	if (unlikely(bio_flagged(bio, BIO_CLONED)))
		return 0;

748
	if (((bio->bi_iter.bi_size + len) >> 9) > queue_max_hw_sectors(q))
Linus Torvalds's avatar
Linus Torvalds committed
749 750
		return 0;

751 752 753 754 755 756 757 758 759 760 761
	/*
	 * For filesystems with a blocksize smaller than the pagesize
	 * we will often be called with the same page as last time and
	 * a consecutive offset.  Optimize this special case.
	 */
	if (bio->bi_vcnt > 0) {
		struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1];

		if (page == prev->bv_page &&
		    offset == prev->bv_offset + prev->bv_len) {
			prev->bv_len += len;
762
			bio->bi_iter.bi_size += len;
763 764
			goto done;
		}
765 766 767 768 769

		/*
		 * If the queue doesn't support SG gaps and adding this
		 * offset would create a gap, disallow it.
		 */
770
		if (bvec_gap_to_prev(q, prev, offset))
771
			return 0;
772 773 774
	}

	if (bio->bi_vcnt >= bio->bi_max_vecs)
Linus Torvalds's avatar
Linus Torvalds committed
775 776 777
		return 0;

	/*
778 779 780 781 782 783 784 785 786 787 788 789 790 791
	 * setup the new entry, we might clear it again later if we
	 * cannot add the page
	 */
	bvec = &bio->bi_io_vec[bio->bi_vcnt];
	bvec->bv_page = page;
	bvec->bv_len = len;
	bvec->bv_offset = offset;
	bio->bi_vcnt++;
	bio->bi_phys_segments++;
	bio->bi_iter.bi_size += len;

	/*
	 * Perform a recount if the number of segments is greater
	 * than queue_max_segments(q).
Linus Torvalds's avatar
Linus Torvalds committed
792 793
	 */

794
	while (bio->bi_phys_segments > queue_max_segments(q)) {
Linus Torvalds's avatar
Linus Torvalds committed
795 796

		if (retried_segments)
797
			goto failed;
Linus Torvalds's avatar
Linus Torvalds committed
798 799 800 801 802 803

		retried_segments = 1;
		blk_recount_segments(q, bio);
	}

	/* If we may be able to merge these biovecs, force a recount */
804
	if (bio->bi_vcnt > 1 && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec)))
805
		bio_clear_flag(bio, BIO_SEG_VALID);
Linus Torvalds's avatar
Linus Torvalds committed
806

807
 done:
Linus Torvalds's avatar
Linus Torvalds committed
808
	return len;
809 810 811 812 813 814 815 816 817

 failed:
	bvec->bv_page = NULL;
	bvec->bv_len = 0;
	bvec->bv_offset = 0;
	bio->bi_vcnt--;
	bio->bi_iter.bi_size -= len;
	blk_recount_segments(q, bio);
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
818
}
819
EXPORT_SYMBOL(bio_add_pc_page);
820

Linus Torvalds's avatar
Linus Torvalds committed
821 822 823 824 825 826 827
/**
 *	bio_add_page	-	attempt to add page to bio
 *	@bio: destination bio
 *	@page: page to add
 *	@len: vec entry length
 *	@offset: vec entry offset
 *
828 829
 *	Attempt to add a page to the bio_vec maplist. This will only fail
 *	if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio.
Linus Torvalds's avatar
Linus Torvalds committed
830
 */
831 832
int bio_add_page(struct bio *bio, struct page *page,
		 unsigned int len, unsigned int offset)
Linus Torvalds's avatar
Linus Torvalds committed
833
{
834 835 836 837 838 839 840
	struct bio_vec *bv;

	/*
	 * cloned bio must not modify vec list
	 */
	if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
		return 0;
841

842 843 844 845 846 847 848
	/*
	 * For filesystems with a blocksize smaller than the pagesize
	 * we will often be called with the same page as last time and
	 * a consecutive offset.  Optimize this special case.
	 */
	if (bio->bi_vcnt > 0) {
		bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
849

850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868
		if (page == bv->bv_page &&
		    offset == bv->bv_offset + bv->bv_len) {
			bv->bv_len += len;
			goto done;
		}
	}

	if (bio->bi_vcnt >= bio->bi_max_vecs)
		return 0;

	bv		= &bio->bi_io_vec[bio->bi_vcnt];
	bv->bv_page	= page;
	bv->bv_len	= len;
	bv->bv_offset	= offset;

	bio->bi_vcnt++;
done:
	bio->bi_iter.bi_size += len;
	return len;
Linus Torvalds's avatar
Linus Torvalds committed
869
}
870
EXPORT_SYMBOL(bio_add_page);
Linus Torvalds's avatar
Linus Torvalds committed
871

872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920
/**
 * bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio
 * @bio: bio to add pages to
 * @iter: iov iterator describing the region to be mapped
 *
 * Pins as many pages from *iter and appends them to @bio's bvec array. The
 * pages will have to be released using put_page() when done.
 */
int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
{
	unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;
	struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;
	struct page **pages = (struct page **)bv;
	size_t offset, diff;
	ssize_t size;

	size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset);
	if (unlikely(size <= 0))
		return size ? size : -EFAULT;
	nr_pages = (size + offset + PAGE_SIZE - 1) / PAGE_SIZE;

	/*
	 * Deep magic below:  We need to walk the pinned pages backwards
	 * because we are abusing the space allocated for the bio_vecs
	 * for the page array.  Because the bio_vecs are larger than the
	 * page pointers by definition this will always work.  But it also
	 * means we can't use bio_add_page, so any changes to it's semantics
	 * need to be reflected here as well.
	 */
	bio->bi_iter.bi_size += size;
	bio->bi_vcnt += nr_pages;

	diff = (nr_pages * PAGE_SIZE - offset) - size;
	while (nr_pages--) {
		bv[nr_pages].bv_page = pages[nr_pages];
		bv[nr_pages].bv_len = PAGE_SIZE;
		bv[nr_pages].bv_offset = 0;
	}

	bv[0].bv_offset += offset;
	bv[0].bv_len -= offset;
	if (diff)
		bv[bio->bi_vcnt - 1].bv_len -= diff;

	iov_iter_advance(iter, size);
	return 0;
}
EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages);

921
static void submit_bio_wait_endio(struct bio *bio)
922
{
923
	complete(bio->bi_private);
924 925 926 927 928 929 930 931
}

/**
 * submit_bio_wait - submit a bio, and wait until it completes
 * @bio: The &struct bio which describes the I/O
 *
 * Simple wrapper around submit_bio(). Returns 0 on success, or the error from
 * bio_endio() on failure.
932 933 934 935
 *
 * WARNING: Unlike to how submit_bio() is usually used, this function does not
 * result in bio reference to be consumed. The caller must drop the reference
 * on his own.
936
 */
937
int submit_bio_wait(struct bio *bio)
938
{
939
	DECLARE_COMPLETION_ONSTACK_MAP(done, bio->bi_disk->lockdep_map);
940

941
	bio->bi_private = &done;
942
	bio->bi_end_io = submit_bio_wait_endio;
Jens Axboe's avatar
Jens Axboe committed
943
	bio->bi_opf |= REQ_SYNC;
944
	submit_bio(bio);
945
	wait_for_completion_io(&done);
946

947
	return blk_status_to_errno(bio->bi_status);
948 949 950
}
EXPORT_SYMBOL(submit_bio_wait);

Kent Overstreet's avatar
Kent Overstreet committed
951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966
/**
 * bio_advance - increment/complete a bio by some number of bytes
 * @bio:	bio to advance
 * @bytes:	number of bytes to complete
 *
 * This updates bi_sector, bi_size and bi_idx; if the number of bytes to
 * complete doesn't align with a bvec boundary, then bv_len and bv_offset will
 * be updated on the last bvec as well.
 *
 * @bio will then represent the remaining, uncompleted portion of the io.
 */
void bio_advance(struct bio *bio, unsigned bytes)
{
	if (bio_integrity(bio))
		bio_integrity_advance(bio, bytes);

Kent Overstreet's avatar
Kent Overstreet committed
967
	bio_advance_iter(bio, &bio->bi_iter, bytes);
Kent Overstreet's avatar
Kent Overstreet committed
968 969 970
}
EXPORT_SYMBOL(bio_advance);

Kent Overstreet's avatar
Kent Overstreet committed
971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998
/**
 * bio_alloc_pages - allocates a single page for each bvec in a bio
 * @bio: bio to allocate pages for
 * @gfp_mask: flags for allocation
 *
 * Allocates pages up to @bio->bi_vcnt.
 *
 * Returns 0 on success, -ENOMEM on failure. On failure, any allocated pages are
 * freed.
 */
int bio_alloc_pages(struct bio *bio, gfp_t gfp_mask)
{
	int i;
	struct bio_vec *bv;

	bio_for_each_segment_all(bv, bio, i) {
		bv->bv_page = alloc_page(gfp_mask);
		if (!bv->bv_page) {
			while (--bv >= bio->bi_io_vec)
				__free_page(bv->bv_page);
			return -ENOMEM;
		}
	}

	return 0;
}
EXPORT_SYMBOL(bio_alloc_pages);

Kent Overstreet's avatar
Kent Overstreet committed
999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012
/**
 * bio_copy_data - copy contents of data buffers from one chain of bios to
 * another
 * @src: source bio list
 * @dst: destination bio list
 *
 * If @src and @dst are single bios, bi_next must be NULL - otherwise, treats
 * @src and @dst as linked lists of bios.
 *
 * Stops when it reaches the end of either @src or @dst - that is, copies
 * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios).
 */
void bio_copy_data(struct bio *dst, struct bio *src)
{
1013 1014
	struct bvec_iter src_iter, dst_iter;
	struct bio_vec src_bv, dst_bv;
Kent Overstreet's avatar
Kent Overstreet committed
1015
	void *src_p, *dst_p;
1016
	unsigned bytes;
Kent Overstreet's avatar
Kent Overstreet committed
1017

1018 1019
	src_iter = src->bi_iter;
	dst_iter = dst->bi_iter;
Kent Overstreet's avatar
Kent Overstreet committed
1020 1021

	while (1) {
1022 1023 1024 1025
		if (!src_iter.bi_size) {
			src = src->bi_next;
			if (!src)
				break;
Kent Overstreet's avatar
Kent Overstreet committed
1026

1027
			src_iter = src->bi_iter;
Kent Overstreet's avatar
Kent Overstreet committed
1028 1029
		}

1030 1031 1032 1033
		if (!dst_iter.bi_size) {
			dst = dst->bi_next;
			if (!dst)
				break;
Kent Overstreet's avatar
Kent Overstreet committed
1034

1035
			dst_iter = dst->bi_iter;
Kent Overstreet's avatar
Kent Overstreet committed
1036 1037
		}

1038 1039 1040 1041
		src_bv = bio_iter_iovec(src, src_iter);
		dst_bv = bio_iter_iovec(dst, dst_iter);

		bytes = min(src_bv.bv_len, dst_bv.bv_len);
Kent Overstreet's avatar
Kent Overstreet committed
1042

1043 1044
		src_p = kmap_atomic(src_bv.bv_page);
		dst_p = kmap_atomic(dst_bv.bv_page);
Kent Overstreet's avatar
Kent Overstreet committed
1045

1046 1047
		memcpy(dst_p + dst_bv.bv_offset,
		       src_p + src_bv.bv_offset,
Kent Overstreet's avatar
Kent Overstreet committed
1048 1049 1050 1051 1052
		       bytes);

		kunmap_atomic(dst_p);
		kunmap_atomic(src_p);

1053 1054
		bio_advance_iter(src, &src_iter, bytes);
		bio_advance_iter(dst, &dst_iter, bytes);
Kent Overstreet's avatar
Kent Overstreet committed
1055 1056 1057 1058
	}
}
EXPORT_SYMBOL(bio_copy_data);

Linus Torvalds's avatar
Linus Torvalds committed
1059
struct bio_map_data {
1060
	int is_our_pages;
1061 1062
	struct iov_iter iter;
	struct iovec iov[];
Linus Torvalds's avatar
Linus Torvalds committed
1063 1064
};

1065
static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data,
1066
					       gfp_t gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
1067
{
1068 1069
	struct bio_map_data *bmd;
	if (data->nr_segs > UIO_MAXIOV)
1070
		return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
1071

1072 1073 1074 1075 1076 1077 1078 1079
	bmd = kmalloc(sizeof(struct bio_map_data) +
		       sizeof(struct iovec) * data->nr_segs, gfp_mask);
	if (!bmd)
		return NULL;
	memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs);
	bmd->iter = *data;
	bmd->iter.iov = bmd->iov;
	return bmd;
Linus Torvalds's avatar
Linus Torvalds committed
1080 1081
}

1082 1083 1084 1085 1086 1087 1088 1089
/**
 * bio_copy_from_iter - copy all pages from iov_iter to bio
 * @bio: The &struct bio which describes the I/O as destination
 * @iter: iov_iter as source
 *
 * Copy all pages from iov_iter to bio.
 * Returns 0 on success, or error on failure.
 */
1090
static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)
1091
{
1092
	int i;
1093 1094
	struct bio_vec *bvec;

1095
	bio_for_each_segment_all(bvec, bio, i) {
1096
		ssize_t ret;
1097

1098 1099 1100
		ret = copy_page_from_iter(bvec->bv_page,
					  bvec->bv_offset,
					  bvec->bv_len,
1101
					  iter);
1102

1103
		if (!iov_iter_count(iter))
1104 1105 1106 1107
			break;

		if (ret < bvec->bv_len)
			return -EFAULT;
1108 1109
	}

1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141
	return 0;
}

/**
 * bio_copy_to_iter - copy all pages from bio to iov_iter
 * @bio: The &struct bio which describes the I/O as source
 * @iter: iov_iter as destination
 *
 * Copy all pages from bio to iov_iter.
 * Returns 0 on success, or error on failure.
 */
static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter)
{
	int i;
	struct bio_vec *bvec;

	bio_for_each_segment_all(bvec, bio, i) {
		ssize_t ret;

		ret = copy_page_to_iter(bvec->bv_page,
					bvec->bv_offset,
					bvec->bv_len,
					&iter);

		if (!iov_iter_count(&iter))
			break;

		if (ret < bvec->bv_len)
			return -EFAULT;
	}

	return 0;
1142 1143
}

1144
void bio_free_pages(struct bio *bio)
1145 1146 1147 1148 1149 1150 1151
{
	struct bio_vec *bvec;
	int i;

	bio_for_each_segment_all(bvec, bio, i)
		__free_page(bvec->bv_page);
}
1152
EXPORT_SYMBOL(bio_free_pages);
1153

Linus Torvalds's avatar
Linus Torvalds committed
1154 1155 1156 1157
/**
 *	bio_uncopy_user	-	finish previously mapped bio
 *	@bio: bio being terminated
 *
1158
 *	Free pages allocated from bio_copy_user_iov() and write back data
Linus Torvalds's avatar
Linus Torvalds committed
1159 1160 1161 1162 1163
 *	to user space in case of a read.
 */
int bio_uncopy_user(struct bio *bio)
{
	struct bio_map_data *bmd = bio->bi_private;
1164
	int ret = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1165

1166 1167 1168
	if (!bio_flagged(bio, BIO_NULL_MAPPED)) {
		/*
		 * if we're in a workqueue, the request is orphaned, so
1169 1170
		 * don't copy into a random user address space, just free
		 * and return -EINTR so user space doesn't expect any data.
1171
		 */
1172 1173 1174
		if (!current->mm)
			ret = -EINTR;
		else if (bio_data_dir(bio) == READ)
1175
			ret = bio_copy_to_iter(bio, bmd->iter);
1176 1177
		if (bmd->is_our_pages)
			bio_free_pages(bio);
1178
	}
1179
	kfree(bmd);
Linus Torvalds's avatar
Linus Torvalds committed
1180 1181 1182 1183 1184
	bio_put(bio);
	return ret;
}

/**
1185
 *	bio_copy_user_iov	-	copy user data to bio
1186 1187 1188 1189
 *	@q:		destination block queue
 *	@map_data:	pointer to the rq_map_data holding pages (if necessary)
 *	@iter:		iovec iterator
 *	@gfp_mask:	memory allocation flags
Linus Torvalds's avatar
Linus Torvalds committed
1190 1191 1192 1193 1194
 *
 *	Prepares and returns a bio for indirect user io, bouncing data
 *	to/from kernel pages as necessary. Must be paired with
 *	call bio_uncopy_user() on io completion.
 */
1195 1196
struct bio *bio_copy_user_iov(struct request_queue *q,
			      struct rq_map_data *map_data,
1197
			      struct iov_iter *iter,
1198
			      gfp_t gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
1199 1200 1201 1202
{
	struct bio_map_data *bmd;
	struct page *page;
	struct bio *bio;
1203 1204
	int i = 0, ret;
	int nr_pages;
1205
	unsigned int len = iter->count;
Geliang Tang's avatar
Geliang Tang committed
1206
	unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0;
Linus Torvalds's avatar
Linus Torvalds committed
1207

1208
	bmd = bio_alloc_map_data(iter, gfp_mask);
Linus Torvalds's avatar
Linus Torvalds committed
1209 1210 1211
	if (!bmd)
		return ERR_PTR(-ENOMEM);