iov_iter.c 34 KB
Newer Older
1
#include <linux/export.h>
2
#include <linux/bvec.h>
3 4
#include <linux/uio.h>
#include <linux/pagemap.h>
5 6
#include <linux/slab.h>
#include <linux/vmalloc.h>
7
#include <linux/splice.h>
Al Viro's avatar
Al Viro committed
8
#include <net/checksum.h>
9

10 11
#define PIPE_PARANOIA /* for now */

12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39
#define iterate_iovec(i, n, __v, __p, skip, STEP) {	\
	size_t left;					\
	size_t wanted = n;				\
	__p = i->iov;					\
	__v.iov_len = min(n, __p->iov_len - skip);	\
	if (likely(__v.iov_len)) {			\
		__v.iov_base = __p->iov_base + skip;	\
		left = (STEP);				\
		__v.iov_len -= left;			\
		skip += __v.iov_len;			\
		n -= __v.iov_len;			\
	} else {					\
		left = 0;				\
	}						\
	while (unlikely(!left && n)) {			\
		__p++;					\
		__v.iov_len = min(n, __p->iov_len);	\
		if (unlikely(!__v.iov_len))		\
			continue;			\
		__v.iov_base = __p->iov_base;		\
		left = (STEP);				\
		__v.iov_len -= left;			\
		skip = __v.iov_len;			\
		n -= __v.iov_len;			\
	}						\
	n = wanted - n;					\
}

40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
#define iterate_kvec(i, n, __v, __p, skip, STEP) {	\
	size_t wanted = n;				\
	__p = i->kvec;					\
	__v.iov_len = min(n, __p->iov_len - skip);	\
	if (likely(__v.iov_len)) {			\
		__v.iov_base = __p->iov_base + skip;	\
		(void)(STEP);				\
		skip += __v.iov_len;			\
		n -= __v.iov_len;			\
	}						\
	while (unlikely(n)) {				\
		__p++;					\
		__v.iov_len = min(n, __p->iov_len);	\
		if (unlikely(!__v.iov_len))		\
			continue;			\
		__v.iov_base = __p->iov_base;		\
		(void)(STEP);				\
		skip = __v.iov_len;			\
		n -= __v.iov_len;			\
	}						\
	n = wanted;					\
}

63 64 65 66 67 68 69
#define iterate_bvec(i, n, __v, __bi, skip, STEP) {	\
	struct bvec_iter __start;			\
	__start.bi_size = n;				\
	__start.bi_bvec_done = skip;			\
	__start.bi_idx = 0;				\
	for_each_bvec(__v, i->bvec, __bi, __start) {	\
		if (!__v.bv_len)			\
70 71 72 73 74
			continue;			\
		(void)(STEP);				\
	}						\
}

75
#define iterate_all_kinds(i, n, v, I, B, K) {			\
76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
	if (likely(n)) {					\
		size_t skip = i->iov_offset;			\
		if (unlikely(i->type & ITER_BVEC)) {		\
			struct bio_vec v;			\
			struct bvec_iter __bi;			\
			iterate_bvec(i, n, v, __bi, skip, (B))	\
		} else if (unlikely(i->type & ITER_KVEC)) {	\
			const struct kvec *kvec;		\
			struct kvec v;				\
			iterate_kvec(i, n, v, kvec, skip, (K))	\
		} else {					\
			const struct iovec *iov;		\
			struct iovec v;				\
			iterate_iovec(i, n, v, iov, skip, (I))	\
		}						\
91 92 93
	}							\
}

94
#define iterate_and_advance(i, n, v, I, B, K) {			\
95 96
	if (unlikely(i->count < n))				\
		n = i->count;					\
97
	if (i->count) {						\
98 99
		size_t skip = i->iov_offset;			\
		if (unlikely(i->type & ITER_BVEC)) {		\
100
			const struct bio_vec *bvec = i->bvec;	\
101
			struct bio_vec v;			\
102 103 104 105 106
			struct bvec_iter __bi;			\
			iterate_bvec(i, n, v, __bi, skip, (B))	\
			i->bvec = __bvec_iter_bvec(i->bvec, __bi);	\
			i->nr_segs -= i->bvec - bvec;		\
			skip = __bi.bi_bvec_done;		\
107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
		} else if (unlikely(i->type & ITER_KVEC)) {	\
			const struct kvec *kvec;		\
			struct kvec v;				\
			iterate_kvec(i, n, v, kvec, skip, (K))	\
			if (skip == kvec->iov_len) {		\
				kvec++;				\
				skip = 0;			\
			}					\
			i->nr_segs -= kvec - i->kvec;		\
			i->kvec = kvec;				\
		} else {					\
			const struct iovec *iov;		\
			struct iovec v;				\
			iterate_iovec(i, n, v, iov, skip, (I))	\
			if (skip == iov->iov_len) {		\
				iov++;				\
				skip = 0;			\
			}					\
			i->nr_segs -= iov - i->iov;		\
			i->iov = iov;				\
127
		}						\
128 129
		i->count -= n;					\
		i->iov_offset = skip;				\
130 131 132
	}							\
}

133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150
static int copyout(void __user *to, const void *from, size_t n)
{
	if (access_ok(VERIFY_WRITE, to, n)) {
		kasan_check_read(from, n);
		n = raw_copy_to_user(to, from, n);
	}
	return n;
}

static int copyin(void *to, const void __user *from, size_t n)
{
	if (access_ok(VERIFY_READ, from, n)) {
		kasan_check_write(to, n);
		n = raw_copy_from_user(to, from, n);
	}
	return n;
}

Al Viro's avatar
Al Viro committed
151
static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
152 153 154 155 156 157 158 159 160 161 162 163 164
			 struct iov_iter *i)
{
	size_t skip, copy, left, wanted;
	const struct iovec *iov;
	char __user *buf;
	void *kaddr, *from;

	if (unlikely(bytes > i->count))
		bytes = i->count;

	if (unlikely(!bytes))
		return 0;

165
	might_fault();
166 167 168 169 170 171
	wanted = bytes;
	iov = i->iov;
	skip = i->iov_offset;
	buf = iov->iov_base + skip;
	copy = min(bytes, iov->iov_len - skip);

172
	if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {
173 174 175 176
		kaddr = kmap_atomic(page);
		from = kaddr + offset;

		/* first chunk, usually the only one */
177
		left = copyout(buf, from, copy);
178 179 180 181 182 183 184 185 186
		copy -= left;
		skip += copy;
		from += copy;
		bytes -= copy;

		while (unlikely(!left && bytes)) {
			iov++;
			buf = iov->iov_base;
			copy = min(bytes, iov->iov_len);
187
			left = copyout(buf, from, copy);
188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
			copy -= left;
			skip = copy;
			from += copy;
			bytes -= copy;
		}
		if (likely(!bytes)) {
			kunmap_atomic(kaddr);
			goto done;
		}
		offset = from - kaddr;
		buf += copy;
		kunmap_atomic(kaddr);
		copy = min(bytes, iov->iov_len - skip);
	}
	/* Too bad - revert to non-atomic kmap */
203

204 205
	kaddr = kmap(page);
	from = kaddr + offset;
206
	left = copyout(buf, from, copy);
207 208 209 210 211 212 213 214
	copy -= left;
	skip += copy;
	from += copy;
	bytes -= copy;
	while (unlikely(!left && bytes)) {
		iov++;
		buf = iov->iov_base;
		copy = min(bytes, iov->iov_len);
215
		left = copyout(buf, from, copy);
216 217 218 219 220 221
		copy -= left;
		skip = copy;
		from += copy;
		bytes -= copy;
	}
	kunmap(page);
222

223
done:
224 225 226 227
	if (skip == iov->iov_len) {
		iov++;
		skip = 0;
	}
228 229 230 231 232 233 234
	i->count -= wanted - bytes;
	i->nr_segs -= iov - i->iov;
	i->iov = iov;
	i->iov_offset = skip;
	return wanted - bytes;
}

Al Viro's avatar
Al Viro committed
235
static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
236 237 238 239 240 241 242 243 244 245 246 247 248
			 struct iov_iter *i)
{
	size_t skip, copy, left, wanted;
	const struct iovec *iov;
	char __user *buf;
	void *kaddr, *to;

	if (unlikely(bytes > i->count))
		bytes = i->count;

	if (unlikely(!bytes))
		return 0;

249
	might_fault();
250 251 252 253 254 255
	wanted = bytes;
	iov = i->iov;
	skip = i->iov_offset;
	buf = iov->iov_base + skip;
	copy = min(bytes, iov->iov_len - skip);

256
	if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {
257 258 259 260
		kaddr = kmap_atomic(page);
		to = kaddr + offset;

		/* first chunk, usually the only one */
261
		left = copyin(to, buf, copy);
262 263 264 265 266 267 268 269 270
		copy -= left;
		skip += copy;
		to += copy;
		bytes -= copy;

		while (unlikely(!left && bytes)) {
			iov++;
			buf = iov->iov_base;
			copy = min(bytes, iov->iov_len);
271
			left = copyin(to, buf, copy);
272 273 274 275 276 277 278 279 280 281 282 283 284 285 286
			copy -= left;
			skip = copy;
			to += copy;
			bytes -= copy;
		}
		if (likely(!bytes)) {
			kunmap_atomic(kaddr);
			goto done;
		}
		offset = to - kaddr;
		buf += copy;
		kunmap_atomic(kaddr);
		copy = min(bytes, iov->iov_len - skip);
	}
	/* Too bad - revert to non-atomic kmap */
287

288 289
	kaddr = kmap(page);
	to = kaddr + offset;
290
	left = copyin(to, buf, copy);
291 292 293 294 295 296 297 298
	copy -= left;
	skip += copy;
	to += copy;
	bytes -= copy;
	while (unlikely(!left && bytes)) {
		iov++;
		buf = iov->iov_base;
		copy = min(bytes, iov->iov_len);
299
		left = copyin(to, buf, copy);
300 301 302 303 304 305
		copy -= left;
		skip = copy;
		to += copy;
		bytes -= copy;
	}
	kunmap(page);
306

307
done:
308 309 310 311
	if (skip == iov->iov_len) {
		iov++;
		skip = 0;
	}
312 313 314 315 316 317 318
	i->count -= wanted - bytes;
	i->nr_segs -= iov - i->iov;
	i->iov = iov;
	i->iov_offset = skip;
	return wanted - bytes;
}

319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
#ifdef PIPE_PARANOIA
static bool sanity(const struct iov_iter *i)
{
	struct pipe_inode_info *pipe = i->pipe;
	int idx = i->idx;
	int next = pipe->curbuf + pipe->nrbufs;
	if (i->iov_offset) {
		struct pipe_buffer *p;
		if (unlikely(!pipe->nrbufs))
			goto Bad;	// pipe must be non-empty
		if (unlikely(idx != ((next - 1) & (pipe->buffers - 1))))
			goto Bad;	// must be at the last buffer...

		p = &pipe->bufs[idx];
		if (unlikely(p->offset + p->len != i->iov_offset))
			goto Bad;	// ... at the end of segment
	} else {
		if (idx != (next & (pipe->buffers - 1)))
			goto Bad;	// must be right after the last buffer
	}
	return true;
Bad:
	printk(KERN_ERR "idx = %d, offset = %zd\n", i->idx, i->iov_offset);
	printk(KERN_ERR "curbuf = %d, nrbufs = %d, buffers = %d\n",
			pipe->curbuf, pipe->nrbufs, pipe->buffers);
	for (idx = 0; idx < pipe->buffers; idx++)
		printk(KERN_ERR "[%p %p %d %d]\n",
			pipe->bufs[idx].ops,
			pipe->bufs[idx].page,
			pipe->bufs[idx].offset,
			pipe->bufs[idx].len);
	WARN_ON(1);
	return false;
}
#else
#define sanity(i) true
#endif

static inline int next_idx(int idx, struct pipe_inode_info *pipe)
{
	return (idx + 1) & (pipe->buffers - 1);
}

static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
			 struct iov_iter *i)
{
	struct pipe_inode_info *pipe = i->pipe;
	struct pipe_buffer *buf;
	size_t off;
	int idx;

	if (unlikely(bytes > i->count))
		bytes = i->count;

	if (unlikely(!bytes))
		return 0;

	if (!sanity(i))
		return 0;

	off = i->iov_offset;
	idx = i->idx;
	buf = &pipe->bufs[idx];
	if (off) {
		if (offset == off && buf->page == page) {
			/* merge with the last one */
			buf->len += bytes;
			i->iov_offset += bytes;
			goto out;
		}
		idx = next_idx(idx, pipe);
		buf = &pipe->bufs[idx];
	}
	if (idx == pipe->curbuf && pipe->nrbufs)
		return 0;
	pipe->nrbufs++;
	buf->ops = &page_cache_pipe_buf_ops;
	get_page(buf->page = page);
	buf->offset = offset;
	buf->len = bytes;
	i->iov_offset = offset + bytes;
	i->idx = idx;
out:
	i->count -= bytes;
	return bytes;
}

406 407 408 409 410 411 412
/*
 * Fault in one or more iovecs of the given iov_iter, to a maximum length of
 * bytes.  For each iovec, fault in each page that constitutes the iovec.
 *
 * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
 * because it is an invalid address).
 */
413
int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
414 415 416 417 418 419 420 421
{
	size_t skip = i->iov_offset;
	const struct iovec *iov;
	int err;
	struct iovec v;

	if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
		iterate_iovec(i, bytes, v, iov, skip, ({
422
			err = fault_in_pages_readable(v.iov_base, v.iov_len);
423 424 425 426 427 428
			if (unlikely(err))
			return err;
		0;}))
	}
	return 0;
}
429
EXPORT_SYMBOL(iov_iter_fault_in_readable);
430

431 432 433 434 435
void iov_iter_init(struct iov_iter *i, int direction,
			const struct iovec *iov, unsigned long nr_segs,
			size_t count)
{
	/* It will get better.  Eventually... */
Al Viro's avatar
Al Viro committed
436
	if (uaccess_kernel()) {
Al Viro's avatar
Al Viro committed
437
		direction |= ITER_KVEC;
438 439 440 441 442 443
		i->type = direction;
		i->kvec = (struct kvec *)iov;
	} else {
		i->type = direction;
		i->iov = iov;
	}
444 445 446 447 448
	i->nr_segs = nr_segs;
	i->iov_offset = 0;
	i->count = count;
}
EXPORT_SYMBOL(iov_iter_init);
449

Al Viro's avatar
Al Viro committed
450 451 452 453 454 455 456
static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
{
	char *from = kmap_atomic(page);
	memcpy(to, from + offset, len);
	kunmap_atomic(from);
}

457
static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
Al Viro's avatar
Al Viro committed
458 459 460 461 462 463
{
	char *to = kmap_atomic(page);
	memcpy(to + offset, from, len);
	kunmap_atomic(to);
}

464 465 466 467 468 469 470
static void memzero_page(struct page *page, size_t offset, size_t len)
{
	char *addr = kmap_atomic(page);
	memset(addr + offset, 0, len);
	kunmap_atomic(addr);
}

471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557
static inline bool allocated(struct pipe_buffer *buf)
{
	return buf->ops == &default_pipe_buf_ops;
}

static inline void data_start(const struct iov_iter *i, int *idxp, size_t *offp)
{
	size_t off = i->iov_offset;
	int idx = i->idx;
	if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) {
		idx = next_idx(idx, i->pipe);
		off = 0;
	}
	*idxp = idx;
	*offp = off;
}

static size_t push_pipe(struct iov_iter *i, size_t size,
			int *idxp, size_t *offp)
{
	struct pipe_inode_info *pipe = i->pipe;
	size_t off;
	int idx;
	ssize_t left;

	if (unlikely(size > i->count))
		size = i->count;
	if (unlikely(!size))
		return 0;

	left = size;
	data_start(i, &idx, &off);
	*idxp = idx;
	*offp = off;
	if (off) {
		left -= PAGE_SIZE - off;
		if (left <= 0) {
			pipe->bufs[idx].len += size;
			return size;
		}
		pipe->bufs[idx].len = PAGE_SIZE;
		idx = next_idx(idx, pipe);
	}
	while (idx != pipe->curbuf || !pipe->nrbufs) {
		struct page *page = alloc_page(GFP_USER);
		if (!page)
			break;
		pipe->nrbufs++;
		pipe->bufs[idx].ops = &default_pipe_buf_ops;
		pipe->bufs[idx].page = page;
		pipe->bufs[idx].offset = 0;
		if (left <= PAGE_SIZE) {
			pipe->bufs[idx].len = left;
			return size;
		}
		pipe->bufs[idx].len = PAGE_SIZE;
		left -= PAGE_SIZE;
		idx = next_idx(idx, pipe);
	}
	return size - left;
}

static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
				struct iov_iter *i)
{
	struct pipe_inode_info *pipe = i->pipe;
	size_t n, off;
	int idx;

	if (!sanity(i))
		return 0;

	bytes = n = push_pipe(i, bytes, &idx, &off);
	if (unlikely(!n))
		return 0;
	for ( ; n; idx = next_idx(idx, pipe), off = 0) {
		size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
		memcpy_to_page(pipe->bufs[idx].page, off, addr, chunk);
		i->idx = idx;
		i->iov_offset = off + chunk;
		n -= chunk;
		addr += chunk;
	}
	i->count -= bytes;
	return bytes;
}

558
size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
Al Viro's avatar
Al Viro committed
559
{
560
	const char *from = addr;
561 562
	if (unlikely(i->type & ITER_PIPE))
		return copy_pipe_to_iter(addr, bytes, i);
563 564
	if (iter_is_iovec(i))
		might_fault();
565
	iterate_and_advance(i, bytes, v,
566
		copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
567
		memcpy_to_page(v.bv_page, v.bv_offset,
568 569
			       (from += v.bv_len) - v.bv_len, v.bv_len),
		memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
570
	)
Al Viro's avatar
Al Viro committed
571

572
	return bytes;
573
}
574
EXPORT_SYMBOL(_copy_to_iter);
575

576
size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
577
{
578
	char *to = addr;
579 580 581 582
	if (unlikely(i->type & ITER_PIPE)) {
		WARN_ON(1);
		return 0;
	}
583 584
	if (iter_is_iovec(i))
		might_fault();
585
	iterate_and_advance(i, bytes, v,
586
		copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
587
		memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
588 589
				 v.bv_offset, v.bv_len),
		memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
590 591 592
	)

	return bytes;
593
}
594
EXPORT_SYMBOL(_copy_from_iter);
595

596
bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
597 598 599 600 601 602
{
	char *to = addr;
	if (unlikely(i->type & ITER_PIPE)) {
		WARN_ON(1);
		return false;
	}
603
	if (unlikely(i->count < bytes))
604 605
		return false;

606 607
	if (iter_is_iovec(i))
		might_fault();
608
	iterate_all_kinds(i, bytes, v, ({
609
		if (copyin((to += v.iov_len) - v.iov_len,
610 611 612 613 614 615 616 617 618 619 620
				      v.iov_base, v.iov_len))
			return false;
		0;}),
		memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
				 v.bv_offset, v.bv_len),
		memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
	)

	iov_iter_advance(i, bytes);
	return true;
}
621
EXPORT_SYMBOL(_copy_from_iter_full);
622

623
size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
Al Viro's avatar
Al Viro committed
624 625
{
	char *to = addr;
626 627 628 629
	if (unlikely(i->type & ITER_PIPE)) {
		WARN_ON(1);
		return 0;
	}
Al Viro's avatar
Al Viro committed
630
	iterate_and_advance(i, bytes, v,
631
		__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
Al Viro's avatar
Al Viro committed
632 633 634 635 636 637 638 639
					 v.iov_base, v.iov_len),
		memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
				 v.bv_offset, v.bv_len),
		memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
	)

	return bytes;
}
640
EXPORT_SYMBOL(_copy_from_iter_nocache);
Al Viro's avatar
Al Viro committed
641

642
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
643
size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660
{
	char *to = addr;
	if (unlikely(i->type & ITER_PIPE)) {
		WARN_ON(1);
		return 0;
	}
	iterate_and_advance(i, bytes, v,
		__copy_from_user_flushcache((to += v.iov_len) - v.iov_len,
					 v.iov_base, v.iov_len),
		memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
				 v.bv_offset, v.bv_len),
		memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base,
			v.iov_len)
	)

	return bytes;
}
661
EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache);
662 663
#endif

664
bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
665 666 667 668 669 670
{
	char *to = addr;
	if (unlikely(i->type & ITER_PIPE)) {
		WARN_ON(1);
		return false;
	}
671
	if (unlikely(i->count < bytes))
672 673
		return false;
	iterate_all_kinds(i, bytes, v, ({
674
		if (__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
675 676 677 678 679 680 681 682 683 684 685
					     v.iov_base, v.iov_len))
			return false;
		0;}),
		memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
				 v.bv_offset, v.bv_len),
		memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
	)

	iov_iter_advance(i, bytes);
	return true;
}
686
EXPORT_SYMBOL(_copy_from_iter_full_nocache);
687

688 689
static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)
{
690 691 692 693 694 695 696 697 698 699 700 701 702 703 704
	struct page *head;
	size_t v = n + offset;

	/*
	 * The general case needs to access the page order in order
	 * to compute the page size.
	 * However, we mostly deal with order-0 pages and thus can
	 * avoid a possible cache line miss for requests that fit all
	 * page orders.
	 */
	if (n <= v && v <= PAGE_SIZE)
		return true;

	head = compound_head(page);
	v += (page - head) << PAGE_SHIFT;
705 706

	if (likely(n <= v && v <= (PAGE_SIZE << compound_order(head))))
707 708 709 710
		return true;
	WARN_ON(1);
	return false;
}
711

Al Viro's avatar
Al Viro committed
712 713 714
size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
			 struct iov_iter *i)
{
715 716
	if (unlikely(!page_copy_sane(page, offset, bytes)))
		return 0;
717 718 719 720 721
	if (i->type & (ITER_BVEC|ITER_KVEC)) {
		void *kaddr = kmap_atomic(page);
		size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
		kunmap_atomic(kaddr);
		return wanted;
722
	} else if (likely(!(i->type & ITER_PIPE)))
Al Viro's avatar
Al Viro committed
723
		return copy_page_to_iter_iovec(page, offset, bytes, i);
724 725
	else
		return copy_page_to_iter_pipe(page, offset, bytes, i);
Al Viro's avatar
Al Viro committed
726 727 728 729 730 731
}
EXPORT_SYMBOL(copy_page_to_iter);

size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
			 struct iov_iter *i)
{
732 733
	if (unlikely(!page_copy_sane(page, offset, bytes)))
		return 0;
734 735 736 737
	if (unlikely(i->type & ITER_PIPE)) {
		WARN_ON(1);
		return 0;
	}
738
	if (i->type & (ITER_BVEC|ITER_KVEC)) {
739
		void *kaddr = kmap_atomic(page);
740
		size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);
741 742 743
		kunmap_atomic(kaddr);
		return wanted;
	} else
Al Viro's avatar
Al Viro committed
744 745 746 747
		return copy_page_from_iter_iovec(page, offset, bytes, i);
}
EXPORT_SYMBOL(copy_page_from_iter);

748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771
static size_t pipe_zero(size_t bytes, struct iov_iter *i)
{
	struct pipe_inode_info *pipe = i->pipe;
	size_t n, off;
	int idx;

	if (!sanity(i))
		return 0;

	bytes = n = push_pipe(i, bytes, &idx, &off);
	if (unlikely(!n))
		return 0;

	for ( ; n; idx = next_idx(idx, pipe), off = 0) {
		size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
		memzero_page(pipe->bufs[idx].page, off, chunk);
		i->idx = idx;
		i->iov_offset = off + chunk;
		n -= chunk;
	}
	i->count -= bytes;
	return bytes;
}

772 773
size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
{
774 775
	if (unlikely(i->type & ITER_PIPE))
		return pipe_zero(bytes, i);
776
	iterate_and_advance(i, bytes, v,
777
		clear_user(v.iov_base, v.iov_len),
778 779
		memzero_page(v.bv_page, v.bv_offset, v.bv_len),
		memset(v.iov_base, 0, v.iov_len)
780 781 782
	)

	return bytes;
783 784 785
}
EXPORT_SYMBOL(iov_iter_zero);

Al Viro's avatar
Al Viro committed
786 787 788
size_t iov_iter_copy_from_user_atomic(struct page *page,
		struct iov_iter *i, unsigned long offset, size_t bytes)
{
789
	char *kaddr = kmap_atomic(page), *p = kaddr + offset;
790 791 792 793
	if (unlikely(!page_copy_sane(page, offset, bytes))) {
		kunmap_atomic(kaddr);
		return 0;
	}
794 795 796 797 798
	if (unlikely(i->type & ITER_PIPE)) {
		kunmap_atomic(kaddr);
		WARN_ON(1);
		return 0;
	}
799
	iterate_all_kinds(i, bytes, v,
800
		copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
801
		memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
802 803
				 v.bv_offset, v.bv_len),
		memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
804 805 806
	)
	kunmap_atomic(kaddr);
	return bytes;
Al Viro's avatar
Al Viro committed
807 808 809
}
EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);

810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829
static inline void pipe_truncate(struct iov_iter *i)
{
	struct pipe_inode_info *pipe = i->pipe;
	if (pipe->nrbufs) {
		size_t off = i->iov_offset;
		int idx = i->idx;
		int nrbufs = (idx - pipe->curbuf) & (pipe->buffers - 1);
		if (off) {
			pipe->bufs[idx].len = off - pipe->bufs[idx].offset;
			idx = next_idx(idx, pipe);
			nrbufs++;
		}
		while (pipe->nrbufs > nrbufs) {
			pipe_buf_release(pipe, &pipe->bufs[idx]);
			idx = next_idx(idx, pipe);
			pipe->nrbufs--;
		}
	}
}

830 831 832 833 834 835
static void pipe_advance(struct iov_iter *i, size_t size)
{
	struct pipe_inode_info *pipe = i->pipe;
	if (unlikely(i->count < size))
		size = i->count;
	if (size) {
836 837 838
		struct pipe_buffer *buf;
		size_t off = i->iov_offset, left = size;
		int idx = i->idx;
839
		if (off) /* make it relative to the beginning of buffer */
840
			left += off - pipe->bufs[idx].offset;
841 842
		while (1) {
			buf = &pipe->bufs[idx];
843
			if (left <= buf->len)
844
				break;
845
			left -= buf->len;
846 847 848
			idx = next_idx(idx, pipe);
		}
		i->idx = idx;
849
		i->iov_offset = buf->offset + left;
850
	}
851 852 853
	i->count -= size;
	/* ... and discard everything past that point */
	pipe_truncate(i);
854 855
}

Al Viro's avatar
Al Viro committed
856 857
void iov_iter_advance(struct iov_iter *i, size_t size)
{
858 859 860 861
	if (unlikely(i->type & ITER_PIPE)) {
		pipe_advance(i, size);
		return;
	}
862
	iterate_and_advance(i, size, v, 0, 0, 0)
Al Viro's avatar
Al Viro committed
863 864 865
}
EXPORT_SYMBOL(iov_iter_advance);

866 867 868 869
void iov_iter_revert(struct iov_iter *i, size_t unroll)
{
	if (!unroll)
		return;
870 871
	if (WARN_ON(unroll > MAX_RW_COUNT))
		return;
872 873 874 875 876 877 878 879
	i->count += unroll;
	if (unlikely(i->type & ITER_PIPE)) {
		struct pipe_inode_info *pipe = i->pipe;
		int idx = i->idx;
		size_t off = i->iov_offset;
		while (1) {
			size_t n = off - pipe->bufs[idx].offset;
			if (unroll < n) {
880
				off -= unroll;
881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929
				break;
			}
			unroll -= n;
			if (!unroll && idx == i->start_idx) {
				off = 0;
				break;
			}
			if (!idx--)
				idx = pipe->buffers - 1;
			off = pipe->bufs[idx].offset + pipe->bufs[idx].len;
		}
		i->iov_offset = off;
		i->idx = idx;
		pipe_truncate(i);
		return;
	}
	if (unroll <= i->iov_offset) {
		i->iov_offset -= unroll;
		return;
	}
	unroll -= i->iov_offset;
	if (i->type & ITER_BVEC) {
		const struct bio_vec *bvec = i->bvec;
		while (1) {
			size_t n = (--bvec)->bv_len;
			i->nr_segs++;
			if (unroll <= n) {
				i->bvec = bvec;
				i->iov_offset = n - unroll;
				return;
			}
			unroll -= n;
		}
	} else { /* same logics for iovec and kvec */
		const struct iovec *iov = i->iov;
		while (1) {
			size_t n = (--iov)->iov_len;
			i->nr_segs++;
			if (unroll <= n) {
				i->iov = iov;
				i->iov_offset = n - unroll;
				return;
			}
			unroll -= n;
		}
	}
}
EXPORT_SYMBOL(iov_iter_revert);

Al Viro's avatar
Al Viro committed
930 931 932 933 934
/*
 * Return the count of just the current iov_iter segment.
 */
size_t iov_iter_single_seg_count(const struct iov_iter *i)
{
935 936
	if (unlikely(i->type & ITER_PIPE))
		return i->count;	// it is a silly place, anyway
Al Viro's avatar
Al Viro committed
937 938 939 940
	if (i->nr_segs == 1)
		return i->count;
	else if (i->type & ITER_BVEC)
		return min(i->count, i->bvec->bv_len - i->iov_offset);
941 942
	else
		return min(i->count, i->iov->iov_len - i->iov_offset);
Al Viro's avatar
Al Viro committed
943 944 945
}
EXPORT_SYMBOL(iov_iter_single_seg_count);

Al Viro's avatar
Al Viro committed
946
void iov_iter_kvec(struct iov_iter *i, int direction,
Al Viro's avatar
Al Viro committed
947
			const struct kvec *kvec, unsigned long nr_segs,
Al Viro's avatar
Al Viro committed
948 949 950 951
			size_t count)
{
	BUG_ON(!(direction & ITER_KVEC));
	i->type = direction;
Al Viro's avatar
Al Viro committed
952
	i->kvec = kvec;
Al Viro's avatar
Al Viro committed
953 954 955 956 957 958
	i->nr_segs = nr_segs;
	i->iov_offset = 0;
	i->count = count;
}
EXPORT_SYMBOL(iov_iter_kvec);

Al Viro's avatar
Al Viro committed
959 960 961 962 963 964 965 966 967 968 969 970 971
void iov_iter_bvec(struct iov_iter *i, int direction,
			const struct bio_vec *bvec, unsigned long nr_segs,
			size_t count)
{
	BUG_ON(!(direction & ITER_BVEC));
	i->type = direction;
	i->bvec = bvec;
	i->nr_segs = nr_segs;
	i->iov_offset = 0;
	i->count = count;
}
EXPORT_SYMBOL(iov_iter_bvec);

972 973 974 975 976
void iov_iter_pipe(struct iov_iter *i, int direction,
			struct pipe_inode_info *pipe,
			size_t count)
{
	BUG_ON(direction != ITER_PIPE);
977
	WARN_ON(pipe->nrbufs == pipe->buffers);
978 979 980 981 982
	i->type = direction;
	i->pipe = pipe;
	i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
	i->iov_offset = 0;
	i->count = count;
983
	i->start_idx = i->idx;
984 985 986
}
EXPORT_SYMBOL(iov_iter_pipe);

Al Viro's avatar
Al Viro committed
987 988
unsigned long iov_iter_alignment(const struct iov_iter *i)
{
989 990 991
	unsigned long res = 0;
	size_t size = i->count;

992
	if (unlikely(i->type & ITER_PIPE)) {
993
		if (size && i->iov_offset && allocated(&i->pipe->bufs[i->idx]))
994 995 996
			return size | i->iov_offset;
		return size;
	}
997 998
	iterate_all_kinds(i, size, v,
		(res |= (unsigned long)v.iov_base | v.iov_len, 0),
999 1000
		res |= v.bv_offset | v.bv_len,
		res |= (unsigned long)v.iov_base | v.iov_len
1001 1002
	)
	return res;
Al Viro's avatar
Al Viro committed
1003 1004 1005
}
EXPORT_SYMBOL(iov_iter_alignment);

1006 1007
unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
{
1008
	unsigned long res = 0;
1009 1010
	size_t size = i->count;

1011 1012 1013 1014 1015
	if (unlikely(i->type & ITER_PIPE)) {
		WARN_ON(1);
		return ~0U;
	}

1016 1017 1018 1019 1020 1021 1022 1023
	iterate_all_kinds(i, size, v,
		(res |= (!res ? 0 : (unsigned long)v.iov_base) |
			(size != v.iov_len ? size : 0), 0),
		(res |= (!res ? 0 : (unsigned long)v.bv_offset) |
			(size != v.bv_len ? size : 0)),
		(res |= (!res ? 0 : (unsigned long)v.iov_base) |
			(size != v.iov_len ? size : 0))
		);
1024
	return res;
1025 1026 1027
}
EXPORT_SYMBOL(iov_iter_gap_alignment);

1028 1029 1030 1031 1032 1033 1034
static inline size_t __pipe_get_pages(struct iov_iter *i,
				size_t maxsize,
				struct page **pages,
				int idx,
				size_t *start)
{
	struct pipe_inode_info *pipe = i->pipe;
1035
	ssize_t n = push_pipe(i, maxsize, &idx, start);
1036 1037 1038 1039 1040
	if (!n)
		return -EFAULT;

	maxsize = n;
	n += *start;
1041
	while (n > 0) {
1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057
		get_page(*pages++ = pipe->bufs[idx].page);
		idx = next_idx(idx, pipe);
		n -= PAGE_SIZE;
	}

	return maxsize;
}

static ssize_t pipe_get_pages(struct iov_iter *i,
		   struct page **pages, size_t maxsize, unsigned maxpages,
		   size_t *start)
{
	unsigned npages;
	size_t capacity;
	int idx;

1058 1059 1060
	if (!maxsize)
		return 0;

1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071
	if (!sanity(i))
		return -EFAULT;

	data_start(i, &idx, start);
	/* some of this one + all after this one */
	npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
	capacity = min(npages,maxpages) * PAGE_SIZE - *start;

	return __pipe_get_pages(i, min(maxsize, capacity), pages, idx, start);
}

Al Viro's avatar
Al Viro committed
1072
ssize_t iov_iter_get_pages(struct iov_iter *i,
1073
		   struct page **pages, size_t maxsize, unsigned maxpages,
Al Viro's avatar
Al Viro committed
1074 1075
		   size_t *start)
{
1076 1077 1078
	if (maxsize > i->count)
		maxsize = i->count;

1079 1080
	if (unlikely(i->type & ITER_PIPE))
		return pipe_get_pages(i, pages, maxsize, maxpages, start);
1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099
	iterate_all_kinds(i, maxsize, v, ({
		unsigned long addr = (unsigned long)v.iov_base;
		size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
		int n;
		int res;

		if (len > maxpages * PAGE_SIZE)
			len = maxpages * PAGE_SIZE;
		addr &= ~(PAGE_SIZE - 1);
		n = DIV_ROUND_UP(len, PAGE_SIZE);
		res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
		if (unlikely(res < 0))
			return res;
		return (res == n ? len : res * PAGE_SIZE) - *start;
	0;}),({
		/* can't be more than PAGE_SIZE */
		*start = v.bv_offset;
		get_page(*pages = v.bv_page);
		return v.bv_len;
1100 1101
	}),({
		return -EFAULT;
1102 1103 1104
	})
	)
	return 0;
Al Viro's avatar
Al Viro committed
1105 1106 1107
}
EXPORT_SYMBOL(iov_iter_get_pages);

1108 1109
static struct page **get_pages_array(size_t n)
{
1110
	return kvmalloc_array(n, sizeof(struct page *), GFP_KERNEL);
1111 1112
}

1113 1114 1115 1116 1117 1118 1119 1120 1121
static ssize_t pipe_get_pages_alloc(struct iov_iter *i,
		   struct page ***pages, size_t maxsize,
		   size_t *start)
{
	struct page **p;
	size_t n;
	int idx;
	int npages;

1122 1123 1124
	if (!maxsize)
		return 0;

1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146
	if (!sanity(i))
		return -EFAULT;

	data_start(i, &idx, start);
	/* some of this one + all after this one */
	npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
	n = npages * PAGE_SIZE - *start;
	if (maxsize > n)
		maxsize = n;
	else
		npages = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
	p = get_pages_array(npages);
	if (!p)
		return -ENOMEM;
	n = __pipe_get_pages(i, maxsize, p, idx, start);
	if (n > 0)
		*pages = p;
	else
		kvfree(p);
	return n;
}

Al Viro's avatar
Al Viro committed
1147 1148 1149 1150
ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
		   struct page ***pages, size_t maxsize,
		   size_t *start)
{
1151 1152 1153 1154 1155
	struct page **p;

	if (maxsize > i->count)
		maxsize = i->count;

1156 1157
	if (unlikely(i->type & ITER_PIPE))
		return pipe_get_pages_alloc(i, pages, maxsize, start);
1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183
	iterate_all_kinds(i, maxsize, v, ({
		unsigned long addr = (unsigned long)v.iov_base;
		size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
		int n;
		int res;

		addr &= ~(PAGE_SIZE - 1);
		n = DIV_ROUND_UP(len, PAGE_SIZE);
		p = get_pages_array(n);
		if (!p)
			return -ENOMEM;
		res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
		if (unlikely(res < 0)) {
			kvfree(p);
			return res;
		}
		*pages = p;
		return (res == n ? len : res * PAGE_SIZE) - *start;
	0;}),({
		/* can't be more than PAGE_SIZE */
		*start = v.bv_offset;
		*pages = p = get_pages_array(1);
		if (!p)
			return -ENOMEM;
		get_page(*p = v.bv_page);
		return v.bv_len;
1184 1185
	}),({
		return -EFAULT;
1186 1187 1188
	})
	)
	return 0;
Al Viro's avatar
Al Viro committed
1189 1190 1191
}
EXPORT_SYMBOL(iov_iter_get_pages_alloc);

Al Viro's avatar
Al Viro committed
1192 1193 1194 1195 1196 1197 1198
size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
			       struct iov_iter *i)
{
	char *to = addr;
	__wsum sum, next;
	size_t off = 0;
	sum = *csum;
1199 1200 1201 1202
	if (unlikely(i->type & ITER_PIPE)) {
		WARN_ON(1);
		return 0;
	}
Al Viro's avatar
Al Viro committed
1203 1204
	iterate_and_advance(i, bytes, v, ({
		int err = 0;
1205
		next = csum_and_copy_from_user(v.iov_base,
Al Viro's avatar
Al Viro committed
1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233
					       (to += v.iov_len) - v.iov_len,
					       v.iov_len, 0, &err);
		if (!err) {
			sum = csum_block_add(sum, next, off);
			off += v.iov_len;
		}
		err ? v.iov_len : 0;
	}), ({
		char *p = kmap_atomic(v.bv_page);
		next = csum_partial_copy_nocheck(p + v.bv_offset,
						 (to += v.bv_len) - v.bv_len,
						 v.bv_len, 0);
		kunmap_atomic(p);
		sum = csum_block_add(sum, next, off);
		off += v.bv_len;
	}),({
		next = csum_partial_copy_nocheck(v.iov_base,
						 (to += v.iov_len) - v.iov_len,
						 v.iov_len, 0);
		sum = csum_block_add(sum, next, off);
		off += v.iov_len;
	})
	)
	*csum = sum;
	return bytes;
}
EXPORT_SYMBOL(csum_and_copy_from_iter);

1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278
bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,
			       struct iov_iter *i)
{
	char *to = addr;
	__wsum sum, next;
	size_t off = 0;
	sum = *csum;
	if (unlikely(i->type & ITER_PIPE)) {
		WARN_ON(1);
		return false;
	}
	if (unlikely(i->count < bytes))
		return false;
	iterate_all_kinds(i, bytes, v, ({
		int err = 0;
		next = csum_and_copy_from_user(v.iov_base,
					       (to += v.iov_len) - v.iov_len,
					       v.iov_len, 0, &err);
		if (err)
			return false;
		sum = csum_block_add(sum, next, off);
		off += v.iov_len;
		0;
	}), ({
		char *p = kmap_atomic(v.bv_page);
		next = csum_partial_copy_nocheck(p + v.bv_offset,
						 (to += v.bv_len) - v.bv_len,
						 v.bv_len, 0);
		kunmap_atomic(p);
		sum = csum_block_add(sum, next, off);
		off += v.bv_len;
	}),({
		next = csum_partial_copy_nocheck(v.iov_base,
						 (to += v.iov_len) - v.iov_len,
						 v.iov_len, 0);
		sum = csum_block_add(sum, next, off);
		off += v.iov_len;
	})
	)
	*csum = sum;
	iov_iter_advance(i, bytes);
	return true;
}
EXPORT_SYMBOL(csum_and_copy_from_iter_full);

1279
size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
Al Viro's avatar
Al Viro committed
1280 1281
			     struct iov_iter *i)
{
1282
	const char *from = addr;
Al Viro's avatar
Al Viro committed
1283 1284 1285
	__wsum sum, next;
	size_t off = 0;
	sum = *csum;
1286 1287 1288 1289
	if (unlikely(i->type & ITER_PIPE)) {
		WARN_ON(1);	/* for now */
		return 0;
	}
Al Viro's avatar
Al Viro committed
1290 1291 1292
	iterate_and_advance(i, bytes, v, ({
		int err = 0;
		next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
1293
					     v.iov_base,
Al Viro's avatar
Al Viro committed
1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320
					     v.iov_len, 0, &err);
		if (!err) {
			sum = csum_block_add(sum, next, off);
			off += v.iov_len;
		}
		err ? v.iov_len : 0;
	}), ({
		char *p = kmap_atomic(v.bv_page);
		next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
						 p + v.bv_offset,
						 v.bv_len, 0);
		kunmap_atomic(p);
		sum = csum_block_add(sum, next, off);
		off += v.bv_len;
	}),({
		next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
						 v.iov_base,
						 v.iov_len, 0);
		sum = csum_block_add(sum, next, off);
		off += v.iov_len;
	})
	)
	*csum = sum;
	return bytes;
}
EXPORT_SYMBOL(csum_and_copy_to_iter);

Al Viro's avatar
Al Viro committed
1321 1322
int iov_iter_npages(const struct iov_iter *i, int maxpages)
{
1323 1324 1325 1326 1327 1328
	size_t size = i->count;
	int npages = 0;

	if (!size)
		return 0;

1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342
	if (unlikely(i->type & ITER_PIPE)) {
		struct pipe_inode_info *pipe = i->pipe;
		size_t off;
		int idx;

		if (!sanity(i))
			return 0;

		data_start(i, &idx, &off);
		/* some of this one + all after this one */
		npages = ((pipe->curbuf - idx - 1) & (pipe->buffers - 1)) + 1;
		if (npages >= maxpages)
			return maxpages;
	} else iterate_all_kinds(i, size, v, ({
1343 1344 1345 1346 1347 1348 1349 1350 1351
		unsigned long p = (unsigned long)v.iov_base;
		npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
			- p / PAGE_SIZE;
		if (npages >= maxpages)
			return maxpages;
	0;}),({
		npages++;
		if (npages >= maxpages)
			return maxpages;
1352 1353 1354 1355 1356 1357
	}),({
		unsigned long p = (unsigned long)v.iov_base;
		npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
			- p / PAGE_SIZE;
		if (npages >= maxpages)
			return maxpages;
1358 1359 1360
	})
	)
	return npages;
Al Viro's avatar
Al Viro committed
1361
}
Al Viro's avatar
Al Viro committed
1362
EXPORT_SYMBOL(iov_iter_npages);
Al Viro's avatar
Al Viro committed
1363 1364 1365 1366

const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
{
	*new = *old;
1367 1368 1369 1370
	if (unlikely(new->type & ITER_PIPE)) {
		WARN_ON(1);
		return NULL;
	}
Al Viro's avatar
Al Viro committed
1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381
	if (new->type & ITER_BVEC)
		return new->bvec = kmemdup(new->bvec,
				    new->nr_segs * sizeof(struct bio_vec),
				    flags);
	else
		/* iovec and kvec have identical layout */
		return new->iov = kmemdup(new->iov,
				   new->nr_segs * sizeof(struct iovec),
				   flags);
}
EXPORT_SYMBOL(dup_iter);
1382

1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404
/**
 * import_iovec() - Copy an array of &struct iovec from userspace
 *     into the kernel, check that it is valid, and initialize a new
 *     &struct iov_iter iterator to access it.
 *
 * @type: One of %READ or %WRITE.
 * @uvector: Pointer to the userspace array.
 * @nr_segs: Number of elements in userspace array.
 * @fast_segs: Number of elements in @iov.
 * @iov: (input and output parameter) Pointer to pointer to (usually small
 *     on-stack) kernel array.
 * @i: Pointer to iterator that will be initialized on success.
 *
 * If the array pointed to by *@iov is large enough to hold all @nr_segs,
 * then this function places %NULL in *@iov on return. Otherwise, a new
 * array will be allocated and the result placed in *@iov. This means that
 * the caller may call kfree() on *@iov regardless of whether the small
 * on-stack array was used or not (and regardless of whether this function
 * returns an error or not).
 *
 * Return: 0 on success or negative error code on error.
 */
1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460
int import_iovec(int type, const struct iovec __user * uvector,
		 unsigned nr_segs, unsigned fast_segs,
		 struct iovec **iov, struct iov_iter *i)
{
	ssize_t n;
	struct iovec *p;
	n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
				  *iov, &p);
	if (n < 0) {
		if (p != *iov)
			kfree(p);
		*iov = NULL;
		return n;
	}
	iov_iter_init(i, type, p, nr_segs, n);
	*iov = p == *iov ? NULL : p;
	return 0;
}
EXPORT_SYMBOL(import_iovec);

#ifdef CONFIG_COMPAT
#include <linux/compat.h>

int compat_import_iovec(int type, const struct compat_iovec __user * uvector,
		 unsigned nr_segs, unsigned fast_segs,
		 struct iovec **iov, struct iov_iter *i)
{
	ssize_t n;
	struct iovec *p;
	n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
				  *iov, &p);
	if (n < 0) {
		if (p != *iov)
			kfree(p);
		*iov = NULL;
		return n;
	}
	iov_iter_init(i, type, p, nr_segs, n);
	*iov = p == *iov ? NULL : p;
	return 0;
}
#endif

int import_single_range(int rw, void __user *buf, size_t len,
		 struct iovec *iov, struct iov_iter *i)
{
	if (len > MAX_RW_COUNT)
		len = MAX_RW_COUNT;
	if (unlikely(!access_ok(!rw, buf, len)))
		return -EFAULT;

	iov->iov_base = buf;
	iov->iov_len = len;
	iov_iter_init(i, rw, iov, 1, len);
	return 0;
}
1461
EXPORT_SYMBOL(import_single_range);