forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/lib/iov_iter.c
....@@ -1,11 +1,17 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
2
+#include <crypto/hash.h>
13 #include <linux/export.h>
24 #include <linux/bvec.h>
5
+#include <linux/fault-inject-usercopy.h>
36 #include <linux/uio.h>
47 #include <linux/pagemap.h>
58 #include <linux/slab.h>
69 #include <linux/vmalloc.h>
710 #include <linux/splice.h>
11
+#include <linux/compat.h>
812 #include <net/checksum.h>
13
+#include <linux/scatterlist.h>
14
+#include <linux/instrumented.h>
915 
1016 #define PIPE_PARANOIA /* for now */
1117 
....@@ -83,6 +89,7 @@
8389 			const struct kvec *kvec;		\
8490 			struct kvec v;				\
8591 			iterate_kvec(i, n, v, kvec, skip, (K))	\
92
+		} else if (unlikely(i->type & ITER_DISCARD)) {	\
8693 		} else {					\
8794 			const struct iovec *iov;		\
8895 			struct iovec v;				\
....@@ -114,6 +121,8 @@
114121 			}					\
115122 			i->nr_segs -= kvec - i->kvec;		\
116123 			i->kvec = kvec;				\
124
+		} else if (unlikely(i->type & ITER_DISCARD)) {	\
125
+			skip += n;				\
117126 		} else {					\
118127 			const struct iovec *iov;		\
119128 			struct iovec v;				\
....@@ -132,8 +141,10 @@
132141 
133142 static int copyout(void __user *to, const void *from, size_t n)
134143 {
135
-	if (access_ok(VERIFY_WRITE, to, n)) {
136
-		kasan_check_read(from, n);
144
+	if (should_fail_usercopy())
145
+		return n;
146
+	if (access_ok(to, n)) {
147
+		instrument_copy_to_user(to, from, n);
137148 		n = raw_copy_to_user(to, from, n);
138149 	}
139150 	return n;
....@@ -141,8 +152,10 @@
141152 
142153 static int copyin(void *to, const void __user *from, size_t n)
143154 {
144
-	if (access_ok(VERIFY_READ, from, n)) {
145
-		kasan_check_write(to, n);
155
+	if (should_fail_usercopy())
156
+		return n;
157
+	if (access_ok(from, n)) {
158
+		instrument_copy_from_user(to, from, n);
146159 		n = raw_copy_from_user(to, from, n);
147160 	}
148161 	return n;
....@@ -320,28 +333,33 @@
320333 static bool sanity(const struct iov_iter *i)
321334 {
322335 	struct pipe_inode_info *pipe = i->pipe;
323
-	int idx = i->idx;
324
-	int next = pipe->curbuf + pipe->nrbufs;
336
+	unsigned int p_head = pipe->head;
337
+	unsigned int p_tail = pipe->tail;
338
+	unsigned int p_mask = pipe->ring_size - 1;
339
+	unsigned int p_occupancy = pipe_occupancy(p_head, p_tail);
340
+	unsigned int i_head = i->head;
341
+	unsigned int idx;
342
+
325343 	if (i->iov_offset) {
326344 		struct pipe_buffer *p;
327
-		if (unlikely(!pipe->nrbufs))
345
+		if (unlikely(p_occupancy == 0))
328346 			goto Bad;	// pipe must be non-empty
329
-		if (unlikely(idx != ((next - 1) & (pipe->buffers - 1))))
347
+		if (unlikely(i_head != p_head - 1))
330348 			goto Bad;	// must be at the last buffer...
331349 
332
-		p = &pipe->bufs[idx];
350
+		p = &pipe->bufs[i_head & p_mask];
333351 		if (unlikely(p->offset + p->len != i->iov_offset))
334352 			goto Bad;	// ... at the end of segment
335353 	} else {
336
-		if (idx != (next & (pipe->buffers - 1)))
354
+		if (i_head != p_head)
337355 			goto Bad;	// must be right after the last buffer
338356 	}
339357 	return true;
340358 Bad:
341
-	printk(KERN_ERR "idx = %d, offset = %zd\n", i->idx, i->iov_offset);
342
-	printk(KERN_ERR "curbuf = %d, nrbufs = %d, buffers = %d\n",
343
-			pipe->curbuf, pipe->nrbufs, pipe->buffers);
344
-	for (idx = 0; idx < pipe->buffers; idx++)
359
+	printk(KERN_ERR "idx = %d, offset = %zd\n", i_head, i->iov_offset);
360
+	printk(KERN_ERR "head = %d, tail = %d, buffers = %d\n",
361
+			p_head, p_tail, pipe->ring_size);
362
+	for (idx = 0; idx < pipe->ring_size; idx++)
345363 		printk(KERN_ERR "[%p %p %d %d]\n",
346364 			pipe->bufs[idx].ops,
347365 			pipe->bufs[idx].page,
....@@ -354,18 +372,15 @@
354372 #define sanity(i) true
355373 #endif
356374 
357
-static inline int next_idx(int idx, struct pipe_inode_info *pipe)
358
-{
359
-	return (idx + 1) & (pipe->buffers - 1);
360
-}
361
-
362375 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
363376 			 struct iov_iter *i)
364377 {
365378 	struct pipe_inode_info *pipe = i->pipe;
366379 	struct pipe_buffer *buf;
380
+	unsigned int p_tail = pipe->tail;
381
+	unsigned int p_mask = pipe->ring_size - 1;
382
+	unsigned int i_head = i->head;
367383 	size_t off;
368
-	int idx;
369384 
370385 	if (unlikely(bytes > i->count))
371386 		bytes = i->count;
....@@ -377,8 +392,7 @@
377392 		return 0;
378393 
379394 	off = i->iov_offset;
380
-	idx = i->idx;
381
-	buf = &pipe->bufs[idx];
395
+	buf = &pipe->bufs[i_head & p_mask];
382396 	if (off) {
383397 		if (offset == off && buf->page == page) {
384398 			/* merge with the last one */
....@@ -386,19 +400,22 @@
386400 			i->iov_offset += bytes;
387401 			goto out;
388402 		}
389
-		idx = next_idx(idx, pipe);
390
-		buf = &pipe->bufs[idx];
403
+		i_head++;
404
+		buf = &pipe->bufs[i_head & p_mask];
391405 	}
392
-	if (idx == pipe->curbuf && pipe->nrbufs)
406
+	if (pipe_full(i_head, p_tail, pipe->max_usage))
393407 		return 0;
394
-	pipe->nrbufs++;
408
+
395409 	buf->ops = &page_cache_pipe_buf_ops;
396410 	buf->flags = 0;
397
-	get_page(buf->page = page);
411
+	get_page(page);
412
+	buf->page = page;
398413 	buf->offset = offset;
399414 	buf->len = bytes;
415
+
416
+	pipe->head = i_head + 1;
400417 	i->iov_offset = offset + bytes;
401
-	i->idx = idx;
418
+	i->head = i_head;
402419 out:
403420 	i->count -= bytes;
404421 	return bytes;
....@@ -429,17 +446,19 @@
429446 }
430447 EXPORT_SYMBOL(iov_iter_fault_in_readable);
431448 
432
-void iov_iter_init(struct iov_iter *i, int direction,
449
+void iov_iter_init(struct iov_iter *i, unsigned int direction,
433450 			const struct iovec *iov, unsigned long nr_segs,
434451 			size_t count)
435452 {
453
+	WARN_ON(direction & ~(READ | WRITE));
454
+	direction &= READ | WRITE;
455
+
436456 	/* It will get better.  Eventually... */
437457 	if (uaccess_kernel()) {
438
-		direction |= ITER_KVEC;
439
-		i->type = direction;
458
+		i->type = ITER_KVEC | direction;
440459 		i->kvec = (struct kvec *)iov;
441460 	} else {
442
-		i->type = direction;
461
+		i->type = ITER_IOVEC | direction;
443462 		i->iov = iov;
444463 	}
445464 	i->nr_segs = nr_segs;
....@@ -474,24 +493,30 @@
474493 	return buf->ops == &default_pipe_buf_ops;
475494 }
476495 
477
-static inline void data_start(const struct iov_iter *i, int *idxp, size_t *offp)
496
+static inline void data_start(const struct iov_iter *i,
497
+			      unsigned int *iter_headp, size_t *offp)
478498 {
499
+	unsigned int p_mask = i->pipe->ring_size - 1;
500
+	unsigned int iter_head = i->head;
479501 	size_t off = i->iov_offset;
480
-	int idx = i->idx;
481
-	if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) {
482
-		idx = next_idx(idx, i->pipe);
502
+
503
+	if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) ||
504
+		    off == PAGE_SIZE)) {
505
+		iter_head++;
483506 		off = 0;
484507 	}
485
-	*idxp = idx;
508
+	*iter_headp = iter_head;
486509 	*offp = off;
487510 }
488511 
489512 static size_t push_pipe(struct iov_iter *i, size_t size,
490
-			int *idxp, size_t *offp)
513
+			int *iter_headp, size_t *offp)
491514 {
492515 	struct pipe_inode_info *pipe = i->pipe;
516
+	unsigned int p_tail = pipe->tail;
517
+	unsigned int p_mask = pipe->ring_size - 1;
518
+	unsigned int iter_head;
493519 	size_t off;
494
-	int idx;
495520 	ssize_t left;
496521 
497522 	if (unlikely(size > i->count))
....@@ -500,34 +525,35 @@
500525 		return 0;
501526 
502527 	left = size;
503
-	data_start(i, &idx, &off);
504
-	*idxp = idx;
528
+	data_start(i, &iter_head, &off);
529
+	*iter_headp = iter_head;
505530 	*offp = off;
506531 	if (off) {
507532 		left -= PAGE_SIZE - off;
508533 		if (left <= 0) {
509
-			pipe->bufs[idx].len += size;
534
+			pipe->bufs[iter_head & p_mask].len += size;
510535 			return size;
511536 		}
512
-		pipe->bufs[idx].len = PAGE_SIZE;
513
-		idx = next_idx(idx, pipe);
537
+		pipe->bufs[iter_head & p_mask].len = PAGE_SIZE;
538
+		iter_head++;
514539 	}
515
-	while (idx != pipe->curbuf || !pipe->nrbufs) {
540
+	while (!pipe_full(iter_head, p_tail, pipe->max_usage)) {
541
+		struct pipe_buffer *buf = &pipe->bufs[iter_head & p_mask];
516542 		struct page *page = alloc_page(GFP_USER);
517543 		if (!page)
518544 			break;
519
-		pipe->nrbufs++;
520
-		pipe->bufs[idx].ops = &default_pipe_buf_ops;
521
-		pipe->bufs[idx].flags = 0;
522
-		pipe->bufs[idx].page = page;
523
-		pipe->bufs[idx].offset = 0;
524
-		if (left <= PAGE_SIZE) {
525
-			pipe->bufs[idx].len = left;
545
+
546
+		buf->ops = &default_pipe_buf_ops;
547
+		buf->flags = 0;
548
+		buf->page = page;
549
+		buf->offset = 0;
550
+		buf->len = min_t(ssize_t, left, PAGE_SIZE);
551
+		left -= buf->len;
552
+		iter_head++;
553
+		pipe->head = iter_head;
554
+
555
+		if (left == 0)
526556 			return size;
527
-		}
528
-		pipe->bufs[idx].len = PAGE_SIZE;
529
-		left -= PAGE_SIZE;
530
-		idx = next_idx(idx, pipe);
531557 	}
532558 	return size - left;
533559 }
....@@ -536,31 +562,77 @@
536562 				struct iov_iter *i)
537563 {
538564 	struct pipe_inode_info *pipe = i->pipe;
565
+	unsigned int p_mask = pipe->ring_size - 1;
566
+	unsigned int i_head;
539567 	size_t n, off;
540
-	int idx;
541568 
542569 	if (!sanity(i))
543570 		return 0;
544571 
545
-	bytes = n = push_pipe(i, bytes, &idx, &off);
572
+	bytes = n = push_pipe(i, bytes, &i_head, &off);
546573 	if (unlikely(!n))
547574 		return 0;
548
-	for ( ; n; idx = next_idx(idx, pipe), off = 0) {
575
+	do {
549576 		size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
550
-		memcpy_to_page(pipe->bufs[idx].page, off, addr, chunk);
551
-		i->idx = idx;
577
+		memcpy_to_page(pipe->bufs[i_head & p_mask].page, off, addr, chunk);
578
+		i->head = i_head;
552579 		i->iov_offset = off + chunk;
553580 		n -= chunk;
554581 		addr += chunk;
555
-	}
582
+		off = 0;
583
+		i_head++;
584
+	} while (n);
556585 	i->count -= bytes;
586
+	return bytes;
587
+}
588
+
589
+static __wsum csum_and_memcpy(void *to, const void *from, size_t len,
590
+			      __wsum sum, size_t off)
591
+{
592
+	__wsum next = csum_partial_copy_nocheck(from, to, len);
593
+	return csum_block_add(sum, next, off);
594
+}
595
+
596
+static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes,
597
+					 struct csum_state *csstate,
598
+					 struct iov_iter *i)
599
+{
600
+	struct pipe_inode_info *pipe = i->pipe;
601
+	unsigned int p_mask = pipe->ring_size - 1;
602
+	__wsum sum = csstate->csum;
603
+	size_t off = csstate->off;
604
+	unsigned int i_head;
605
+	size_t n, r;
606
+
607
+	if (!sanity(i))
608
+		return 0;
609
+
610
+	bytes = n = push_pipe(i, bytes, &i_head, &r);
611
+	if (unlikely(!n))
612
+		return 0;
613
+	do {
614
+		size_t chunk = min_t(size_t, n, PAGE_SIZE - r);
615
+		char *p = kmap_atomic(pipe->bufs[i_head & p_mask].page);
616
+		sum = csum_and_memcpy(p + r, addr, chunk, sum, off);
617
+		kunmap_atomic(p);
618
+		i->head = i_head;
619
+		i->iov_offset = r + chunk;
620
+		n -= chunk;
621
+		off += chunk;
622
+		addr += chunk;
623
+		r = 0;
624
+		i_head++;
625
+	} while (n);
626
+	i->count -= bytes;
627
+	csstate->csum = sum;
628
+	csstate->off = off;
557629 	return bytes;
558630 }
559631 
560632 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
561633 {
562634 	const char *from = addr;
563
-	if (unlikely(i->type & ITER_PIPE))
635
+	if (unlikely(iov_iter_is_pipe(i)))
564636 		return copy_pipe_to_iter(addr, bytes, i);
565637 	if (iter_is_iovec(i))
566638 		might_fault();
....@@ -575,73 +647,75 @@
575647 }
576648 EXPORT_SYMBOL(_copy_to_iter);
577649 
578
-#ifdef CONFIG_ARCH_HAS_UACCESS_MCSAFE
579
-static int copyout_mcsafe(void __user *to, const void *from, size_t n)
650
+#ifdef CONFIG_ARCH_HAS_COPY_MC
651
+static int copyout_mc(void __user *to, const void *from, size_t n)
580652 {
581
-	if (access_ok(VERIFY_WRITE, to, n)) {
582
-		kasan_check_read(from, n);
583
-		n = copy_to_user_mcsafe((__force void *) to, from, n);
653
+	if (access_ok(to, n)) {
654
+		instrument_copy_to_user(to, from, n);
655
+		n = copy_mc_to_user((__force void *) to, from, n);
584656 	}
585657 	return n;
586658 }
587659 
588
-static unsigned long memcpy_mcsafe_to_page(struct page *page, size_t offset,
660
+static unsigned long copy_mc_to_page(struct page *page, size_t offset,
589661 		const char *from, size_t len)
590662 {
591663 	unsigned long ret;
592664 	char *to;
593665 
594666 	to = kmap_atomic(page);
595
-	ret = memcpy_mcsafe(to + offset, from, len);
667
+	ret = copy_mc_to_kernel(to + offset, from, len);
596668 	kunmap_atomic(to);
597669 
598670 	return ret;
599671 }
600672 
601
-static size_t copy_pipe_to_iter_mcsafe(const void *addr, size_t bytes,
673
+static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes,
602674 				struct iov_iter *i)
603675 {
604676 	struct pipe_inode_info *pipe = i->pipe;
677
+	unsigned int p_mask = pipe->ring_size - 1;
678
+	unsigned int i_head;
605679 	size_t n, off, xfer = 0;
606
-	int idx;
607680 
608681 	if (!sanity(i))
609682 		return 0;
610683 
611
-	bytes = n = push_pipe(i, bytes, &idx, &off);
684
+	bytes = n = push_pipe(i, bytes, &i_head, &off);
612685 	if (unlikely(!n))
613686 		return 0;
614
-	for ( ; n; idx = next_idx(idx, pipe), off = 0) {
687
+	do {
615688 		size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
616689 		unsigned long rem;
617690 
618
-		rem = memcpy_mcsafe_to_page(pipe->bufs[idx].page, off, addr,
619
-				chunk);
620
-		i->idx = idx;
691
+		rem = copy_mc_to_page(pipe->bufs[i_head & p_mask].page,
692
+					    off, addr, chunk);
693
+		i->head = i_head;
621694 		i->iov_offset = off + chunk - rem;
622695 		xfer += chunk - rem;
623696 		if (rem)
624697 			break;
625698 		n -= chunk;
626699 		addr += chunk;
627
-	}
700
+		off = 0;
701
+		i_head++;
702
+	} while (n);
628703 	i->count -= xfer;
629704 	return xfer;
630705 }
631706 
632707 /**
633
- * _copy_to_iter_mcsafe - copy to user with source-read error exception handling
708
+ * _copy_mc_to_iter - copy to iter with source memory error exception handling
634709  * @addr: source kernel address
635710  * @bytes: total transfer length
636711  * @iter: destination iterator
637712  *
638
- * The pmem driver arranges for filesystem-dax to use this facility via
639
- * dax_copy_to_iter() for protecting read/write to persistent memory.
640
- * Unless / until an architecture can guarantee identical performance
641
- * between _copy_to_iter_mcsafe() and _copy_to_iter() it would be a
642
- * performance regression to switch more users to the mcsafe version.
713
+ * The pmem driver deploys this for the dax operation
714
+ * (dax_copy_to_iter()) for dax reads (bypass page-cache and the
715
+ * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
716
+ * successfully copied.
643717  *
644
- * Otherwise, the main differences between this and typical _copy_to_iter().
718
+ * The main differences between this and typical _copy_to_iter().
645719  *
646720  * * Typical tail/residue handling after a fault retries the copy
647721  *   byte-by-byte until the fault happens again. Re-triggering machine
....@@ -652,23 +726,22 @@
652726  * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies.
653727  *   Compare to copy_to_iter() where only ITER_IOVEC attempts might return
654728  *   a short copy.
655
- *
656
- * See MCSAFE_TEST for self-test.
657729  */
658
-size_t _copy_to_iter_mcsafe(const void *addr, size_t bytes, struct iov_iter *i)
730
+size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
659731 {
660732 	const char *from = addr;
661733 	unsigned long rem, curr_addr, s_addr = (unsigned long) addr;
662734 
663
-	if (unlikely(i->type & ITER_PIPE))
664
-		return copy_pipe_to_iter_mcsafe(addr, bytes, i);
735
+	if (unlikely(iov_iter_is_pipe(i)))
736
+		return copy_mc_pipe_to_iter(addr, bytes, i);
665737 	if (iter_is_iovec(i))
666738 		might_fault();
667739 	iterate_and_advance(i, bytes, v,
668
-		copyout_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
740
+		copyout_mc(v.iov_base, (from += v.iov_len) - v.iov_len,
741
+			   v.iov_len),
669742 		({
670
-		rem = memcpy_mcsafe_to_page(v.bv_page, v.bv_offset,
671
-                               (from += v.bv_len) - v.bv_len, v.bv_len);
743
+		rem = copy_mc_to_page(v.bv_page, v.bv_offset,
744
+				      (from += v.bv_len) - v.bv_len, v.bv_len);
672745 		if (rem) {
673746 			curr_addr = (unsigned long) from;
674747 			bytes = curr_addr - s_addr - rem;
....@@ -676,8 +749,8 @@
676749 		}
677750 		}),
678751 		({
679
-		rem = memcpy_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len,
680
-				v.iov_len);
752
+		rem = copy_mc_to_kernel(v.iov_base, (from += v.iov_len)
753
+					- v.iov_len, v.iov_len);
681754 		if (rem) {
682755 			curr_addr = (unsigned long) from;
683756 			bytes = curr_addr - s_addr - rem;
....@@ -688,13 +761,13 @@
688761 
689762 	return bytes;
690763 }
691
-EXPORT_SYMBOL_GPL(_copy_to_iter_mcsafe);
692
-#endif /* CONFIG_ARCH_HAS_UACCESS_MCSAFE */
764
+EXPORT_SYMBOL_GPL(_copy_mc_to_iter);
765
+#endif /* CONFIG_ARCH_HAS_COPY_MC */
693766 
694767 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
695768 {
696769 	char *to = addr;
697
-	if (unlikely(i->type & ITER_PIPE)) {
770
+	if (unlikely(iov_iter_is_pipe(i))) {
698771 		WARN_ON(1);
699772 		return 0;
700773 	}
....@@ -714,7 +787,7 @@
714787 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
715788 {
716789 	char *to = addr;
717
-	if (unlikely(i->type & ITER_PIPE)) {
790
+	if (unlikely(iov_iter_is_pipe(i))) {
718791 		WARN_ON(1);
719792 		return false;
720793 	}
....@@ -741,7 +814,7 @@
741814 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
742815 {
743816 	char *to = addr;
744
-	if (unlikely(i->type & ITER_PIPE)) {
817
+	if (unlikely(iov_iter_is_pipe(i))) {
745818 		WARN_ON(1);
746819 		return 0;
747820 	}
....@@ -775,7 +848,7 @@
775848 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
776849 {
777850 	char *to = addr;
778
-	if (unlikely(i->type & ITER_PIPE)) {
851
+	if (unlikely(iov_iter_is_pipe(i))) {
779852 		WARN_ON(1);
780853 		return 0;
781854 	}
....@@ -796,7 +869,7 @@
796869 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
797870 {
798871 	char *to = addr;
799
-	if (unlikely(i->type & ITER_PIPE)) {
872
+	if (unlikely(iov_iter_is_pipe(i))) {
800873 		WARN_ON(1);
801874 		return false;
802875 	}
....@@ -835,7 +908,7 @@
835908 	head = compound_head(page);
836909 	v += (page - head) << PAGE_SHIFT;
837910 
838
-	if (likely(n <= v && v <= (PAGE_SIZE << compound_order(head))))
911
+	if (likely(n <= v && v <= (page_size(head))))
839912 		return true;
840913 	WARN_ON(1);
841914 	return false;
....@@ -851,7 +924,12 @@
851924 		size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
852925 		kunmap_atomic(kaddr);
853926 		return wanted;
854
-	} else if (likely(!(i->type & ITER_PIPE)))
927
+	} else if (unlikely(iov_iter_is_discard(i))) {
928
+		if (unlikely(i->count < bytes))
929
+			bytes = i->count;
930
+		i->count -= bytes;
931
+		return bytes;
932
+	} else if (likely(!iov_iter_is_pipe(i)))
855933 		return copy_page_to_iter_iovec(page, offset, bytes, i);
856934 	else
857935 		return copy_page_to_iter_pipe(page, offset, bytes, i);
....@@ -863,7 +941,7 @@
863941 {
864942 	if (unlikely(!page_copy_sane(page, offset, bytes)))
865943 		return 0;
866
-	if (unlikely(i->type & ITER_PIPE)) {
944
+	if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
867945 		WARN_ON(1);
868946 		return 0;
869947 	}
....@@ -880,30 +958,33 @@
880958 static size_t pipe_zero(size_t bytes, struct iov_iter *i)
881959 {
882960 	struct pipe_inode_info *pipe = i->pipe;
961
+	unsigned int p_mask = pipe->ring_size - 1;
962
+	unsigned int i_head;
883963 	size_t n, off;
884
-	int idx;
885964 
886965 	if (!sanity(i))
887966 		return 0;
888967 
889
-	bytes = n = push_pipe(i, bytes, &idx, &off);
968
+	bytes = n = push_pipe(i, bytes, &i_head, &off);
890969 	if (unlikely(!n))
891970 		return 0;
892971 
893
-	for ( ; n; idx = next_idx(idx, pipe), off = 0) {
972
+	do {
894973 		size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
895
-		memzero_page(pipe->bufs[idx].page, off, chunk);
896
-		i->idx = idx;
974
+		memzero_page(pipe->bufs[i_head & p_mask].page, off, chunk);
975
+		i->head = i_head;
897976 		i->iov_offset = off + chunk;
898977 		n -= chunk;
899
-	}
978
+		off = 0;
979
+		i_head++;
980
+	} while (n);
900981 	i->count -= bytes;
901982 	return bytes;
902983 }
903984 
904985 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
905986 {
906
-	if (unlikely(i->type & ITER_PIPE))
987
+	if (unlikely(iov_iter_is_pipe(i)))
907988 		return pipe_zero(bytes, i);
908989 	iterate_and_advance(i, bytes, v,
909990 		clear_user(v.iov_base, v.iov_len),
....@@ -923,7 +1004,7 @@
9231004 		kunmap_atomic(kaddr);
9241005 		return 0;
9251006 	}
926
-	if (unlikely(i->type & ITER_PIPE)) {
1007
+	if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
9271008 		kunmap_atomic(kaddr);
9281009 		WARN_ON(1);
9291010 		return 0;
....@@ -942,20 +1023,26 @@
9421023 static inline void pipe_truncate(struct iov_iter *i)
9431024 {
9441025 	struct pipe_inode_info *pipe = i->pipe;
945
-	if (pipe->nrbufs) {
1026
+	unsigned int p_tail = pipe->tail;
1027
+	unsigned int p_head = pipe->head;
1028
+	unsigned int p_mask = pipe->ring_size - 1;
1029
+
1030
+	if (!pipe_empty(p_head, p_tail)) {
1031
+		struct pipe_buffer *buf;
1032
+		unsigned int i_head = i->head;
9461033 		size_t off = i->iov_offset;
947
-		int idx = i->idx;
948
-		int nrbufs = (idx - pipe->curbuf) & (pipe->buffers - 1);
1034
+
9491035 		if (off) {
950
-			pipe->bufs[idx].len = off - pipe->bufs[idx].offset;
951
-			idx = next_idx(idx, pipe);
952
-			nrbufs++;
1036
+			buf = &pipe->bufs[i_head & p_mask];
1037
+			buf->len = off - buf->offset;
1038
+			i_head++;
9531039 		}
954
-		while (pipe->nrbufs > nrbufs) {
955
-			pipe_buf_release(pipe, &pipe->bufs[idx]);
956
-			idx = next_idx(idx, pipe);
957
-			pipe->nrbufs--;
1040
+		while (p_head != i_head) {
1041
+			p_head--;
1042
+			pipe_buf_release(pipe, &pipe->bufs[p_head & p_mask]);
9581043 		}
1044
+
1045
+		pipe->head = p_head;
9591046 	}
9601047 }
9611048 
....@@ -966,18 +1053,20 @@
9661053 		size = i->count;
9671054 	if (size) {
9681055 		struct pipe_buffer *buf;
1056
+		unsigned int p_mask = pipe->ring_size - 1;
1057
+		unsigned int i_head = i->head;
9691058 		size_t off = i->iov_offset, left = size;
970
-		int idx = i->idx;
1059
+
9711060 		if (off) /* make it relative to the beginning of buffer */
972
-			left += off - pipe->bufs[idx].offset;
1061
+			left += off - pipe->bufs[i_head & p_mask].offset;
9731062 		while (1) {
974
-			buf = &pipe->bufs[idx];
1063
+			buf = &pipe->bufs[i_head & p_mask];
9751064 			if (left <= buf->len)
9761065 				break;
9771066 			left -= buf->len;
978
-			idx = next_idx(idx, pipe);
1067
+			i_head++;
9791068 		}
980
-		i->idx = idx;
1069
+		i->head = i_head;
9811070 		i->iov_offset = buf->offset + left;
9821071 	}
9831072 	i->count -= size;
....@@ -987,8 +1076,12 @@
9871076 
9881077 void iov_iter_advance(struct iov_iter *i, size_t size)
9891078 {
990
-	if (unlikely(i->type & ITER_PIPE)) {
1079
+	if (unlikely(iov_iter_is_pipe(i))) {
9911080 		pipe_advance(i, size);
1081
+		return;
1082
+	}
1083
+	if (unlikely(iov_iter_is_discard(i))) {
1084
+		i->count -= size;
9921085 		return;
9931086 	}
9941087 	iterate_and_advance(i, size, v, 0, 0, 0)
....@@ -1002,36 +1095,40 @@
10021095 	if (WARN_ON(unroll > MAX_RW_COUNT))
10031096 		return;
10041097 	i->count += unroll;
1005
-	if (unlikely(i->type & ITER_PIPE)) {
1098
+	if (unlikely(iov_iter_is_pipe(i))) {
10061099 		struct pipe_inode_info *pipe = i->pipe;
1007
-		int idx = i->idx;
1100
+		unsigned int p_mask = pipe->ring_size - 1;
1101
+		unsigned int i_head = i->head;
10081102 		size_t off = i->iov_offset;
10091103 		while (1) {
1010
-			size_t n = off - pipe->bufs[idx].offset;
1104
+			struct pipe_buffer *b = &pipe->bufs[i_head & p_mask];
1105
+			size_t n = off - b->offset;
10111106 			if (unroll < n) {
10121107 				off -= unroll;
10131108 				break;
10141109 			}
10151110 			unroll -= n;
1016
-			if (!unroll && idx == i->start_idx) {
1111
+			if (!unroll && i_head == i->start_head) {
10171112 				off = 0;
10181113 				break;
10191114 			}
1020
-			if (!idx--)
1021
-				idx = pipe->buffers - 1;
1022
-			off = pipe->bufs[idx].offset + pipe->bufs[idx].len;
1115
+			i_head--;
1116
+			b = &pipe->bufs[i_head & p_mask];
1117
+			off = b->offset + b->len;
10231118 		}
10241119 		i->iov_offset = off;
1025
-		i->idx = idx;
1120
+		i->head = i_head;
10261121 		pipe_truncate(i);
10271122 		return;
10281123 	}
1124
+	if (unlikely(iov_iter_is_discard(i)))
1125
+		return;
10291126 	if (unroll <= i->iov_offset) {
10301127 		i->iov_offset -= unroll;
10311128 		return;
10321129 	}
10331130 	unroll -= i->iov_offset;
1034
-	if (i->type & ITER_BVEC) {
1131
+	if (iov_iter_is_bvec(i)) {
10351132 		const struct bio_vec *bvec = i->bvec;
10361133 		while (1) {
10371134 			size_t n = (--bvec)->bv_len;
....@@ -1064,23 +1161,25 @@
10641161  */
10651162 size_t iov_iter_single_seg_count(const struct iov_iter *i)
10661163 {
1067
-	if (unlikely(i->type & ITER_PIPE))
1164
+	if (unlikely(iov_iter_is_pipe(i)))
10681165 		return i->count;	// it is a silly place, anyway
10691166 	if (i->nr_segs == 1)
10701167 		return i->count;
1071
-	else if (i->type & ITER_BVEC)
1168
+	if (unlikely(iov_iter_is_discard(i)))
1169
+		return i->count;
1170
+	else if (iov_iter_is_bvec(i))
10721171 		return min(i->count, i->bvec->bv_len - i->iov_offset);
10731172 	else
10741173 		return min(i->count, i->iov->iov_len - i->iov_offset);
10751174 }
10761175 EXPORT_SYMBOL(iov_iter_single_seg_count);
10771176 
1078
-void iov_iter_kvec(struct iov_iter *i, int direction,
1177
+void iov_iter_kvec(struct iov_iter *i, unsigned int direction,
10791178 			const struct kvec *kvec, unsigned long nr_segs,
10801179 			size_t count)
10811180 {
1082
-	BUG_ON(!(direction & ITER_KVEC));
1083
-	i->type = direction;
1181
+	WARN_ON(direction & ~(READ | WRITE));
1182
+	i->type = ITER_KVEC | (direction & (READ | WRITE));
10841183 	i->kvec = kvec;
10851184 	i->nr_segs = nr_segs;
10861185 	i->iov_offset = 0;
....@@ -1088,12 +1187,12 @@
10881187 }
10891188 EXPORT_SYMBOL(iov_iter_kvec);
10901189 
1091
-void iov_iter_bvec(struct iov_iter *i, int direction,
1190
+void iov_iter_bvec(struct iov_iter *i, unsigned int direction,
10921191 			const struct bio_vec *bvec, unsigned long nr_segs,
10931192 			size_t count)
10941193 {
1095
-	BUG_ON(!(direction & ITER_BVEC));
1096
-	i->type = direction;
1194
+	WARN_ON(direction & ~(READ | WRITE));
1195
+	i->type = ITER_BVEC | (direction & (READ | WRITE));
10971196 	i->bvec = bvec;
10981197 	i->nr_segs = nr_segs;
10991198 	i->iov_offset = 0;
....@@ -1101,28 +1200,48 @@
11011200 }
11021201 EXPORT_SYMBOL(iov_iter_bvec);
11031202 
1104
-void iov_iter_pipe(struct iov_iter *i, int direction,
1203
+void iov_iter_pipe(struct iov_iter *i, unsigned int direction,
11051204 			struct pipe_inode_info *pipe,
11061205 			size_t count)
11071206 {
1108
-	BUG_ON(direction != ITER_PIPE);
1109
-	WARN_ON(pipe->nrbufs == pipe->buffers);
1110
-	i->type = direction;
1207
+	BUG_ON(direction != READ);
1208
+	WARN_ON(pipe_full(pipe->head, pipe->tail, pipe->ring_size));
1209
+	i->type = ITER_PIPE | READ;
11111210 	i->pipe = pipe;
1112
-	i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
1211
+	i->head = pipe->head;
11131212 	i->iov_offset = 0;
11141213 	i->count = count;
1115
-	i->start_idx = i->idx;
1214
+	i->start_head = i->head;
11161215 }
11171216 EXPORT_SYMBOL(iov_iter_pipe);
1217
+
1218
+/**
1219
+ * iov_iter_discard - Initialise an I/O iterator that discards data
1220
+ * @i: The iterator to initialise.
1221
+ * @direction: The direction of the transfer.
1222
+ * @count: The size of the I/O buffer in bytes.
1223
+ *
1224
+ * Set up an I/O iterator that just discards everything that's written to it.
1225
+ * It's only available as a READ iterator.
1226
+ */
1227
+void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count)
1228
+{
1229
+	BUG_ON(direction != READ);
1230
+	i->type = ITER_DISCARD | READ;
1231
+	i->count = count;
1232
+	i->iov_offset = 0;
1233
+}
1234
+EXPORT_SYMBOL(iov_iter_discard);
11181235 
11191236 unsigned long iov_iter_alignment(const struct iov_iter *i)
11201237 {
11211238 	unsigned long res = 0;
11221239 	size_t size = i->count;
11231240 
1124
-	if (unlikely(i->type & ITER_PIPE)) {
1125
-		if (size && i->iov_offset && allocated(&i->pipe->bufs[i->idx]))
1241
+	if (unlikely(iov_iter_is_pipe(i))) {
1242
+		unsigned int p_mask = i->pipe->ring_size - 1;
1243
+
1244
+		if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask]))
11261245 			return size | i->iov_offset;
11271246 		return size;
11281247 	}
....@@ -1140,7 +1259,7 @@
11401259 	unsigned long res = 0;
11411260 	size_t size = i->count;
11421261 
1143
-	if (unlikely(i->type & ITER_PIPE)) {
1262
+	if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
11441263 		WARN_ON(1);
11451264 		return ~0U;
11461265 	}
....@@ -1160,19 +1279,20 @@
11601279 static inline ssize_t __pipe_get_pages(struct iov_iter *i,
11611280 				size_t maxsize,
11621281 				struct page **pages,
1163
-				int idx,
1282
+				int iter_head,
11641283 				size_t *start)
11651284 {
11661285 	struct pipe_inode_info *pipe = i->pipe;
1167
-	ssize_t n = push_pipe(i, maxsize, &idx, start);
1286
+	unsigned int p_mask = pipe->ring_size - 1;
1287
+	ssize_t n = push_pipe(i, maxsize, &iter_head, start);
11681288 	if (!n)
11691289 		return -EFAULT;
11701290 
11711291 	maxsize = n;
11721292 	n += *start;
11731293 	while (n > 0) {
1174
-		get_page(*pages++ = pipe->bufs[idx].page);
1175
-		idx = next_idx(idx, pipe);
1294
+		get_page(*pages++ = pipe->bufs[iter_head & p_mask].page);
1295
+		iter_head++;
11761296 		n -= PAGE_SIZE;
11771297 	}
11781298 
....@@ -1183,9 +1303,8 @@
11831303 		   struct page **pages, size_t maxsize, unsigned maxpages,
11841304 		   size_t *start)
11851305 {
1186
-	unsigned npages;
1306
+	unsigned int iter_head, npages;
11871307 	size_t capacity;
1188
-	int idx;
11891308 
11901309 	if (!maxsize)
11911310 		return 0;
....@@ -1193,12 +1312,12 @@
11931312 	if (!sanity(i))
11941313 		return -EFAULT;
11951314 
1196
-	data_start(i, &idx, start);
1197
-	/* some of this one + all after this one */
1198
-	npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
1199
-	capacity = min(npages,maxpages) * PAGE_SIZE - *start;
1315
+	data_start(i, &iter_head, start);
1316
+	/* Amount of free space: some of this one + all after this one */
1317
+	npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);
1318
+	capacity = min(npages, maxpages) * PAGE_SIZE - *start;
12001319 
1201
-	return __pipe_get_pages(i, min(maxsize, capacity), pages, idx, start);
1320
+	return __pipe_get_pages(i, min(maxsize, capacity), pages, iter_head, start);
12021321 }
12031322 
12041323 ssize_t iov_iter_get_pages(struct iov_iter *i,
....@@ -1208,8 +1327,11 @@
12081327 	if (maxsize > i->count)
12091328 		maxsize = i->count;
12101329 
1211
-	if (unlikely(i->type & ITER_PIPE))
1330
+	if (unlikely(iov_iter_is_pipe(i)))
12121331 		return pipe_get_pages(i, pages, maxsize, maxpages, start);
1332
+	if (unlikely(iov_iter_is_discard(i)))
1333
+		return -EFAULT;
1334
+
12131335 	iterate_all_kinds(i, maxsize, v, ({
12141336 		unsigned long addr = (unsigned long)v.iov_base;
12151337 		size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
....@@ -1220,8 +1342,10 @@
12201342 			len = maxpages * PAGE_SIZE;
12211343 		addr &= ~(PAGE_SIZE - 1);
12221344 		n = DIV_ROUND_UP(len, PAGE_SIZE);
1223
-		res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
1224
-		if (unlikely(res < 0))
1345
+		res = get_user_pages_fast(addr, n,
1346
+				iov_iter_rw(i) != WRITE ?  FOLL_WRITE : 0,
1347
+				pages);
1348
+		if (unlikely(res <= 0))
12251349 			return res;
12261350 		return (res == n ? len : res * PAGE_SIZE) - *start;
12271351 	0;}),({
....@@ -1247,9 +1371,8 @@
12471371 		   size_t *start)
12481372 {
12491373 	struct page **p;
1374
+	unsigned int iter_head, npages;
12501375 	ssize_t n;
1251
-	int idx;
1252
-	int npages;
12531376 
12541377 	if (!maxsize)
12551378 		return 0;
....@@ -1257,9 +1380,9 @@
12571380 	if (!sanity(i))
12581381 		return -EFAULT;
12591382 
1260
-	data_start(i, &idx, start);
1261
-	/* some of this one + all after this one */
1262
-	npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
1383
+	data_start(i, &iter_head, start);
1384
+	/* Amount of free space: some of this one + all after this one */
1385
+	npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);
12631386 	n = npages * PAGE_SIZE - *start;
12641387 	if (maxsize > n)
12651388 		maxsize = n;
....@@ -1268,7 +1391,7 @@
12681391 	p = get_pages_array(npages);
12691392 	if (!p)
12701393 		return -ENOMEM;
1271
-	n = __pipe_get_pages(i, maxsize, p, idx, start);
1394
+	n = __pipe_get_pages(i, maxsize, p, iter_head, start);
12721395 	if (n > 0)
12731396 		*pages = p;
12741397 	else
....@@ -1285,8 +1408,11 @@
12851408 	if (maxsize > i->count)
12861409 		maxsize = i->count;
12871410 
1288
-	if (unlikely(i->type & ITER_PIPE))
1411
+	if (unlikely(iov_iter_is_pipe(i)))
12891412 		return pipe_get_pages_alloc(i, pages, maxsize, start);
1413
+	if (unlikely(iov_iter_is_discard(i)))
1414
+		return -EFAULT;
1415
+
12901416 	iterate_all_kinds(i, maxsize, v, ({
12911417 		unsigned long addr = (unsigned long)v.iov_base;
12921418 		size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
....@@ -1298,9 +1424,11 @@
12981424 		p = get_pages_array(n);
12991425 		if (!p)
13001426 			return -ENOMEM;
1301
-		res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
1302
-		if (unlikely(res < 0)) {
1427
+		res = get_user_pages_fast(addr, n,
1428
+				iov_iter_rw(i) != WRITE ?  FOLL_WRITE : 0, p);
1429
+		if (unlikely(res <= 0)) {
13031430 			kvfree(p);
1431
+			*pages = NULL;
13041432 			return res;
13051433 		}
13061434 		*pages = p;
....@@ -1328,33 +1456,30 @@
13281456 	__wsum sum, next;
13291457 	size_t off = 0;
13301458 	sum = *csum;
1331
-	if (unlikely(i->type & ITER_PIPE)) {
1459
+	if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
13321460 		WARN_ON(1);
13331461 		return 0;
13341462 	}
13351463 	iterate_and_advance(i, bytes, v, ({
1336
-		int err = 0;
13371464 		next = csum_and_copy_from_user(v.iov_base,
13381465 					       (to += v.iov_len) - v.iov_len,
1339
-					       v.iov_len, 0, &err);
1340
-		if (!err) {
1466
+					       v.iov_len);
1467
+		if (next) {
13411468 			sum = csum_block_add(sum, next, off);
13421469 			off += v.iov_len;
13431470 		}
1344
-		err ? v.iov_len : 0;
1471
+		next ? 0 : v.iov_len;
13451472 	}), ({
13461473 		char *p = kmap_atomic(v.bv_page);
1347
-		next = csum_partial_copy_nocheck(p + v.bv_offset,
1348
-						 (to += v.bv_len) - v.bv_len,
1349
-						 v.bv_len, 0);
1474
+		sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1475
+				      p + v.bv_offset, v.bv_len,
1476
+				      sum, off);
13501477 		kunmap_atomic(p);
1351
-		sum = csum_block_add(sum, next, off);
13521478 		off += v.bv_len;
13531479 	}),({
1354
-		next = csum_partial_copy_nocheck(v.iov_base,
1355
-						 (to += v.iov_len) - v.iov_len,
1356
-						 v.iov_len, 0);
1357
-		sum = csum_block_add(sum, next, off);
1480
+		sum = csum_and_memcpy((to += v.iov_len) - v.iov_len,
1481
+				      v.iov_base, v.iov_len,
1482
+				      sum, off);
13581483 		off += v.iov_len;
13591484 	})
13601485 	)
....@@ -1370,35 +1495,32 @@
13701495 	__wsum sum, next;
13711496 	size_t off = 0;
13721497 	sum = *csum;
1373
-	if (unlikely(i->type & ITER_PIPE)) {
1498
+	if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
13741499 		WARN_ON(1);
13751500 		return false;
13761501 	}
13771502 	if (unlikely(i->count < bytes))
13781503 		return false;
13791504 	iterate_all_kinds(i, bytes, v, ({
1380
-		int err = 0;
13811505 		next = csum_and_copy_from_user(v.iov_base,
13821506 					       (to += v.iov_len) - v.iov_len,
1383
-					       v.iov_len, 0, &err);
1384
-		if (err)
1507
+					       v.iov_len);
1508
+		if (!next)
13851509 			return false;
13861510 		sum = csum_block_add(sum, next, off);
13871511 		off += v.iov_len;
13881512 		0;
13891513 	}), ({
13901514 		char *p = kmap_atomic(v.bv_page);
1391
-		next = csum_partial_copy_nocheck(p + v.bv_offset,
1392
-						 (to += v.bv_len) - v.bv_len,
1393
-						 v.bv_len, 0);
1515
+		sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1516
+				      p + v.bv_offset, v.bv_len,
1517
+				      sum, off);
13941518 		kunmap_atomic(p);
1395
-		sum = csum_block_add(sum, next, off);
13961519 		off += v.bv_len;
13971520 	}),({
1398
-		next = csum_partial_copy_nocheck(v.iov_base,
1399
-						 (to += v.iov_len) - v.iov_len,
1400
-						 v.iov_len, 0);
1401
-		sum = csum_block_add(sum, next, off);
1521
+		sum = csum_and_memcpy((to += v.iov_len) - v.iov_len,
1522
+				      v.iov_base, v.iov_len,
1523
+				      sum, off);
14021524 		off += v.iov_len;
14031525 	})
14041526 	)
....@@ -1408,47 +1530,70 @@
14081530 }
14091531 EXPORT_SYMBOL(csum_and_copy_from_iter_full);
14101532 
1411
-size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
1533
+size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate,
14121534 			     struct iov_iter *i)
14131535 {
1536
+	struct csum_state *csstate = _csstate;
14141537 	const char *from = addr;
14151538 	__wsum sum, next;
1416
-	size_t off = 0;
1417
-	sum = *csum;
1418
-	if (unlikely(i->type & ITER_PIPE)) {
1539
+	size_t off;
1540
+
1541
+	if (unlikely(iov_iter_is_pipe(i)))
1542
+		return csum_and_copy_to_pipe_iter(addr, bytes, _csstate, i);
1543
+
1544
+	sum = csstate->csum;
1545
+	off = csstate->off;
1546
+	if (unlikely(iov_iter_is_discard(i))) {
14191547 		WARN_ON(1);	/* for now */
14201548 		return 0;
14211549 	}
14221550 	iterate_and_advance(i, bytes, v, ({
1423
-		int err = 0;
14241551 		next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
14251552 					     v.iov_base,
1426
-					     v.iov_len, 0, &err);
1427
-		if (!err) {
1553
+					     v.iov_len);
1554
+		if (next) {
14281555 			sum = csum_block_add(sum, next, off);
14291556 			off += v.iov_len;
14301557 		}
1431
-		err ? v.iov_len : 0;
1558
+		next ? 0 : v.iov_len;
14321559 	}), ({
14331560 		char *p = kmap_atomic(v.bv_page);
1434
-		next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
1435
-						 p + v.bv_offset,
1436
-						 v.bv_len, 0);
1561
+		sum = csum_and_memcpy(p + v.bv_offset,
1562
+				      (from += v.bv_len) - v.bv_len,
1563
+				      v.bv_len, sum, off);
14371564 		kunmap_atomic(p);
1438
-		sum = csum_block_add(sum, next, off);
14391565 		off += v.bv_len;
14401566 	}),({
1441
-		next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
1442
-						 v.iov_base,
1443
-						 v.iov_len, 0);
1444
-		sum = csum_block_add(sum, next, off);
1567
+		sum = csum_and_memcpy(v.iov_base,
1568
+				     (from += v.iov_len) - v.iov_len,
1569
+				     v.iov_len, sum, off);
14451570 		off += v.iov_len;
14461571 	})
14471572 	)
1448
-	*csum = sum;
1573
+	csstate->csum = sum;
1574
+	csstate->off = off;
14491575 	return bytes;
14501576 }
14511577 EXPORT_SYMBOL(csum_and_copy_to_iter);
1578
+
1579
+size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
1580
+		struct iov_iter *i)
1581
+{
1582
+#ifdef CONFIG_CRYPTO_HASH
1583
+	struct ahash_request *hash = hashp;
1584
+	struct scatterlist sg;
1585
+	size_t copied;
1586
+
1587
+	copied = copy_to_iter(addr, bytes, i);
1588
+	sg_init_one(&sg, addr, copied);
1589
+	ahash_request_set_crypt(hash, &sg, NULL, copied);
1590
+	crypto_ahash_update(hash);
1591
+	return copied;
1592
+#else
1593
+	return 0;
1594
+#endif
1595
+}
1596
+EXPORT_SYMBOL(hash_and_copy_to_iter);
14521597 
14531598 int iov_iter_npages(const struct iov_iter *i, int maxpages)
14541599 {
....@@ -1457,18 +1602,20 @@
14571602 
14581603 	if (!size)
14591604 		return 0;
1605
+	if (unlikely(iov_iter_is_discard(i)))
1606
+		return 0;
14601607 
1461
-	if (unlikely(i->type & ITER_PIPE)) {
1608
+	if (unlikely(iov_iter_is_pipe(i))) {
14621609 		struct pipe_inode_info *pipe = i->pipe;
1610
+		unsigned int iter_head;
14631611 		size_t off;
1464
-		int idx;
14651612 
14661613 		if (!sanity(i))
14671614 			return 0;
14681615 
1469
-		data_start(i, &idx, &off);
1616
+		data_start(i, &iter_head, &off);
14701617 		/* some of this one + all after this one */
1471
-		npages = ((pipe->curbuf - idx - 1) & (pipe->buffers - 1)) + 1;
1618
+		npages = pipe_space_for_user(iter_head, pipe->tail, pipe);
14721619 		if (npages >= maxpages)
14731620 			return maxpages;
14741621 	} else iterate_all_kinds(i, size, v, ({
....@@ -1496,11 +1643,13 @@
14961643 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
14971644 {
14981645 	*new = *old;
1499
-	if (unlikely(new->type & ITER_PIPE)) {
1646
+	if (unlikely(iov_iter_is_pipe(new))) {
15001647 		WARN_ON(1);
15011648 		return NULL;
15021649 	}
1503
-	if (new->type & ITER_BVEC)
1650
+	if (unlikely(iov_iter_is_discard(new)))
1651
+		return NULL;
1652
+	if (iov_iter_is_bvec(new))
15041653 		return new->bvec = kmemdup(new->bvec,
15051654 				    new->nr_segs * sizeof(struct bio_vec),
15061655 				    flags);
....@@ -1512,16 +1661,145 @@
15121661 }
15131662 EXPORT_SYMBOL(dup_iter);
15141663 
1664
+static int copy_compat_iovec_from_user(struct iovec *iov,
1665
+		const struct iovec __user *uvec, unsigned long nr_segs)
1666
+{
1667
+	const struct compat_iovec __user *uiov =
1668
+		(const struct compat_iovec __user *)uvec;
1669
+	int ret = -EFAULT, i;
1670
+
1671
+	if (!user_access_begin(uiov, nr_segs * sizeof(*uiov)))
1672
+		return -EFAULT;
1673
+
1674
+	for (i = 0; i < nr_segs; i++) {
1675
+		compat_uptr_t buf;
1676
+		compat_ssize_t len;
1677
+
1678
+		unsafe_get_user(len, &uiov[i].iov_len, uaccess_end);
1679
+		unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end);
1680
+
1681
+		/* check for compat_size_t not fitting in compat_ssize_t .. */
1682
+		if (len < 0) {
1683
+			ret = -EINVAL;
1684
+			goto uaccess_end;
1685
+		}
1686
+		iov[i].iov_base = compat_ptr(buf);
1687
+		iov[i].iov_len = len;
1688
+	}
1689
+
1690
+	ret = 0;
1691
+uaccess_end:
1692
+	user_access_end();
1693
+	return ret;
1694
+}
1695
+
1696
+static int copy_iovec_from_user(struct iovec *iov,
1697
+		const struct iovec __user *uvec, unsigned long nr_segs)
1698
+{
1699
+	unsigned long seg;
1700
+
1701
+	if (copy_from_user(iov, uvec, nr_segs * sizeof(*uvec)))
1702
+		return -EFAULT;
1703
+	for (seg = 0; seg < nr_segs; seg++) {
1704
+		if ((ssize_t)iov[seg].iov_len < 0)
1705
+			return -EINVAL;
1706
+	}
1707
+
1708
+	return 0;
1709
+}
1710
+
1711
+struct iovec *iovec_from_user(const struct iovec __user *uvec,
1712
+		unsigned long nr_segs, unsigned long fast_segs,
1713
+		struct iovec *fast_iov, bool compat)
1714
+{
1715
+	struct iovec *iov = fast_iov;
1716
+	int ret;
1717
+
1718
+	/*
1719
+	 * SuS says "The readv() function *may* fail if the iovcnt argument was
1720
+	 * less than or equal to 0, or greater than {IOV_MAX}.  Linux has
1721
+	 * traditionally returned zero for zero segments, so...
1722
+	 */
1723
+	if (nr_segs == 0)
1724
+		return iov;
1725
+	if (nr_segs > UIO_MAXIOV)
1726
+		return ERR_PTR(-EINVAL);
1727
+	if (nr_segs > fast_segs) {
1728
+		iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
1729
+		if (!iov)
1730
+			return ERR_PTR(-ENOMEM);
1731
+	}
1732
+
1733
+	if (compat)
1734
+		ret = copy_compat_iovec_from_user(iov, uvec, nr_segs);
1735
+	else
1736
+		ret = copy_iovec_from_user(iov, uvec, nr_segs);
1737
+	if (ret) {
1738
+		if (iov != fast_iov)
1739
+			kfree(iov);
1740
+		return ERR_PTR(ret);
1741
+	}
1742
+
1743
+	return iov;
1744
+}
1745
+
1746
+ssize_t __import_iovec(int type, const struct iovec __user *uvec,
1747
+		 unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
1748
+		 struct iov_iter *i, bool compat)
1749
+{
1750
+	ssize_t total_len = 0;
1751
+	unsigned long seg;
1752
+	struct iovec *iov;
1753
+
1754
+	iov = iovec_from_user(uvec, nr_segs, fast_segs, *iovp, compat);
1755
+	if (IS_ERR(iov)) {
1756
+		*iovp = NULL;
1757
+		return PTR_ERR(iov);
1758
+	}
1759
+
1760
+	/*
1761
+	 * According to the Single Unix Specification we should return EINVAL if
1762
+	 * an element length is < 0 when cast to ssize_t or if the total length
1763
+	 * would overflow the ssize_t return value of the system call.
1764
+	 *
1765
+	 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
1766
+	 * overflow case.
1767
+	 */
1768
+	for (seg = 0; seg < nr_segs; seg++) {
1769
+		ssize_t len = (ssize_t)iov[seg].iov_len;
1770
+
1771
+		if (!access_ok(iov[seg].iov_base, len)) {
1772
+			if (iov != *iovp)
1773
+				kfree(iov);
1774
+			*iovp = NULL;
1775
+			return -EFAULT;
1776
+		}
1777
+
1778
+		if (len > MAX_RW_COUNT - total_len) {
1779
+			len = MAX_RW_COUNT - total_len;
1780
+			iov[seg].iov_len = len;
1781
+		}
1782
+		total_len += len;
1783
+	}
1784
+
1785
+	iov_iter_init(i, type, iov, nr_segs, total_len);
1786
+	if (iov == *iovp)
1787
+		*iovp = NULL;
1788
+	else
1789
+		*iovp = iov;
1790
+	return total_len;
1791
+}
1792
+
15151793 /**
15161794  * import_iovec() - Copy an array of &struct iovec from userspace
15171795  *     into the kernel, check that it is valid, and initialize a new
15181796  *     &struct iov_iter iterator to access it.
15191797  *
15201798  * @type: One of %READ or %WRITE.
1521
- * @uvector: Pointer to the userspace array.
1799
+ * @uvec: Pointer to the userspace array.
15221800  * @nr_segs: Number of elements in userspace array.
15231801  * @fast_segs: Number of elements in @iov.
1524
- * @iov: (input and output parameter) Pointer to pointer to (usually small
1802
+ * @iovp: (input and output parameter) Pointer to pointer to (usually small
15251803  *     on-stack) kernel array.
15261804  * @i: Pointer to iterator that will be initialized on success.
15271805  *
....@@ -1532,57 +1810,23 @@
15321810  * on-stack array was used or not (and regardless of whether this function
15331811  * returns an error or not).
15341812  *
1535
- * Return: 0 on success or negative error code on error.
1813
+ * Return: Negative error code on error, bytes imported on success
15361814  */
1537
-int import_iovec(int type, const struct iovec __user * uvector,
1815
+ssize_t import_iovec(int type, const struct iovec __user *uvec,
15381816 		 unsigned nr_segs, unsigned fast_segs,
1539
-		 struct iovec **iov, struct iov_iter *i)
1817
+		 struct iovec **iovp, struct iov_iter *i)
15401818 {
1541
-	ssize_t n;
1542
-	struct iovec *p;
1543
-	n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
1544
-				  *iov, &p);
1545
-	if (n < 0) {
1546
-		if (p != *iov)
1547
-			kfree(p);
1548
-		*iov = NULL;
1549
-		return n;
1550
-	}
1551
-	iov_iter_init(i, type, p, nr_segs, n);
1552
-	*iov = p == *iov ? NULL : p;
1553
-	return 0;
1819
+	return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i,
1820
+			      in_compat_syscall());
15541821 }
15551822 EXPORT_SYMBOL(import_iovec);
1556
-
1557
-#ifdef CONFIG_COMPAT
1558
-#include <linux/compat.h>
1559
-
1560
-int compat_import_iovec(int type, const struct compat_iovec __user * uvector,
1561
-		 unsigned nr_segs, unsigned fast_segs,
1562
-		 struct iovec **iov, struct iov_iter *i)
1563
-{
1564
-	ssize_t n;
1565
-	struct iovec *p;
1566
-	n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
1567
-				  *iov, &p);
1568
-	if (n < 0) {
1569
-		if (p != *iov)
1570
-			kfree(p);
1571
-		*iov = NULL;
1572
-		return n;
1573
-	}
1574
-	iov_iter_init(i, type, p, nr_segs, n);
1575
-	*iov = p == *iov ? NULL : p;
1576
-	return 0;
1577
-}
1578
-#endif
15791823 
15801824 int import_single_range(int rw, void __user *buf, size_t len,
15811825 		 struct iovec *iov, struct iov_iter *i)
15821826 {
15831827 	if (len > MAX_RW_COUNT)
15841828 		len = MAX_RW_COUNT;
1585
-	if (unlikely(!access_ok(!rw, buf, len)))
1829
+	if (unlikely(!access_ok(buf, len)))
15861830 		return -EFAULT;
15871831 
15881832 	iov->iov_base = buf;
....@@ -1592,24 +1836,38 @@
15921836 }
15931837 EXPORT_SYMBOL(import_single_range);
15941838 
1595
-int iov_iter_for_each_range(struct iov_iter *i, size_t bytes,
1596
-			    int (*f)(struct kvec *vec, void *context),
1597
-			    void *context)
1839
+/**
1840
+ * iov_iter_restore() - Restore a &struct iov_iter to the same state as when
1841
+ *     iov_iter_save_state() was called.
1842
+ *
1843
+ * @i: &struct iov_iter to restore
1844
+ * @state: state to restore from
1845
+ *
1846
+ * Used after iov_iter_save_state() to bring restore @i, if operations may
1847
+ * have advanced it.
1848
+ *
1849
+ * Note: only works on ITER_IOVEC, ITER_BVEC, and ITER_KVEC
1850
+ */
1851
+void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state)
15981852 {
1599
-	struct kvec w;
1600
-	int err = -EINVAL;
1601
-	if (!bytes)
1602
-		return 0;
1603
-
1604
-	iterate_all_kinds(i, bytes, v, -EINVAL, ({
1605
-		w.iov_base = kmap(v.bv_page) + v.bv_offset;
1606
-		w.iov_len = v.bv_len;
1607
-		err = f(&w, context);
1608
-		kunmap(v.bv_page);
1609
-		err;}), ({
1610
-		w = v;
1611
-		err = f(&w, context);})
1612
-	)
1613
-	return err;
1853
+	if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i)) &&
1854
+			 !iov_iter_is_kvec(i))
1855
+		return;
1856
+	i->iov_offset = state->iov_offset;
1857
+	i->count = state->count;
1858
+	/*
1859
+	 * For the *vec iters, nr_segs + iov is constant - if we increment
1860
+	 * the vec, then we also decrement the nr_segs count. Hence we don't
1861
+	 * need to track both of these, just one is enough and we can deduct
1862
+	 * the other from that. ITER_KVEC and ITER_IOVEC are the same struct
1863
+	 * size, so we can just increment the iov pointer as they are unionzed.
1864
+	 * ITER_BVEC _may_ be the same size on some archs, but on others it is
1865
+	 * not. Be safe and handle it separately.
1866
+	 */
1867
+	BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec));
1868
+	if (iov_iter_is_bvec(i))
1869
+		i->bvec -= state->nr_segs - i->nr_segs;
1870
+	else
1871
+		i->iov -= state->nr_segs - i->nr_segs;
1872
+	i->nr_segs = state->nr_segs;
16141873 }
1615
-EXPORT_SYMBOL(iov_iter_for_each_range);