forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-10-09 244b2c5ca8b14627e4a17755e5922221e121c771 
 kernel/lib/iov_iter.c
....@@ -1,11 +1,17 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
2
+#include <crypto/hash.h>
13 #include <linux/export.h>
24 #include <linux/bvec.h>
5
+#include <linux/fault-inject-usercopy.h>
36 #include <linux/uio.h>
47 #include <linux/pagemap.h>
58 #include <linux/slab.h>
69 #include <linux/vmalloc.h>
710 #include <linux/splice.h>
11
+#include <linux/compat.h>
812 #include <net/checksum.h>
13
+#include <linux/scatterlist.h>
14
+#include <linux/instrumented.h>
915 
1016 #define PIPE_PARANOIA /* for now */
1117 
....@@ -83,6 +89,7 @@
8389 			const struct kvec *kvec;		\
8490 			struct kvec v;				\
8591 			iterate_kvec(i, n, v, kvec, skip, (K))	\
92
+		} else if (unlikely(i->type & ITER_DISCARD)) {	\
8693 		} else {					\
8794 			const struct iovec *iov;		\
8895 			struct iovec v;				\
....@@ -114,6 +121,8 @@
114121 			}					\
115122 			i->nr_segs -= kvec - i->kvec;		\
116123 			i->kvec = kvec;				\
124
+		} else if (unlikely(i->type & ITER_DISCARD)) {	\
125
+			skip += n;				\
117126 		} else {					\
118127 			const struct iovec *iov;		\
119128 			struct iovec v;				\
....@@ -132,8 +141,10 @@
132141 
133142 static int copyout(void __user *to, const void *from, size_t n)
134143 {
135
-	if (access_ok(VERIFY_WRITE, to, n)) {
136
-		kasan_check_read(from, n);
144
+	if (should_fail_usercopy())
145
+		return n;
146
+	if (access_ok(to, n)) {
147
+		instrument_copy_to_user(to, from, n);
137148 		n = raw_copy_to_user(to, from, n);
138149 	}
139150 	return n;
....@@ -141,8 +152,10 @@
141152 
142153 static int copyin(void *to, const void __user *from, size_t n)
143154 {
144
-	if (access_ok(VERIFY_READ, from, n)) {
145
-		kasan_check_write(to, n);
155
+	if (should_fail_usercopy())
156
+		return n;
157
+	if (access_ok(from, n)) {
158
+		instrument_copy_from_user(to, from, n);
146159 		n = raw_copy_from_user(to, from, n);
147160 	}
148161 	return n;
....@@ -320,28 +333,33 @@
320333 static bool sanity(const struct iov_iter *i)
321334 {
322335 	struct pipe_inode_info *pipe = i->pipe;
323
-	int idx = i->idx;
324
-	int next = pipe->curbuf + pipe->nrbufs;
336
+	unsigned int p_head = pipe->head;
337
+	unsigned int p_tail = pipe->tail;
338
+	unsigned int p_mask = pipe->ring_size - 1;
339
+	unsigned int p_occupancy = pipe_occupancy(p_head, p_tail);
340
+	unsigned int i_head = i->head;
341
+	unsigned int idx;
342
+
325343 	if (i->iov_offset) {
326344 		struct pipe_buffer *p;
327
-		if (unlikely(!pipe->nrbufs))
345
+		if (unlikely(p_occupancy == 0))
328346 			goto Bad;	// pipe must be non-empty
329
-		if (unlikely(idx != ((next - 1) & (pipe->buffers - 1))))
347
+		if (unlikely(i_head != p_head - 1))
330348 			goto Bad;	// must be at the last buffer...
331349 
332
-		p = &pipe->bufs[idx];
350
+		p = &pipe->bufs[i_head & p_mask];
333351 		if (unlikely(p->offset + p->len != i->iov_offset))
334352 			goto Bad;	// ... at the end of segment
335353 	} else {
336
-		if (idx != (next & (pipe->buffers - 1)))
354
+		if (i_head != p_head)
337355 			goto Bad;	// must be right after the last buffer
338356 	}
339357 	return true;
340358 Bad:
341
-	printk(KERN_ERR "idx = %d, offset = %zd\n", i->idx, i->iov_offset);
342
-	printk(KERN_ERR "curbuf = %d, nrbufs = %d, buffers = %d\n",
343
-			pipe->curbuf, pipe->nrbufs, pipe->buffers);
344
-	for (idx = 0; idx < pipe->buffers; idx++)
359
+	printk(KERN_ERR "idx = %d, offset = %zd\n", i_head, i->iov_offset);
360
+	printk(KERN_ERR "head = %d, tail = %d, buffers = %d\n",
361
+			p_head, p_tail, pipe->ring_size);
362
+	for (idx = 0; idx < pipe->ring_size; idx++)
345363 		printk(KERN_ERR "[%p %p %d %d]\n",
346364 			pipe->bufs[idx].ops,
347365 			pipe->bufs[idx].page,
....@@ -354,18 +372,15 @@
354372 #define sanity(i) true
355373 #endif
356374 
357
-static inline int next_idx(int idx, struct pipe_inode_info *pipe)
358
-{
359
-	return (idx + 1) & (pipe->buffers - 1);
360
-}
361
-
362375 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
363376 			 struct iov_iter *i)
364377 {
365378 	struct pipe_inode_info *pipe = i->pipe;
366379 	struct pipe_buffer *buf;
380
+	unsigned int p_tail = pipe->tail;
381
+	unsigned int p_mask = pipe->ring_size - 1;
382
+	unsigned int i_head = i->head;
367383 	size_t off;
368
-	int idx;
369384 
370385 	if (unlikely(bytes > i->count))
371386 		bytes = i->count;
....@@ -377,8 +392,7 @@
377392 		return 0;
378393 
379394 	off = i->iov_offset;
380
-	idx = i->idx;
381
-	buf = &pipe->bufs[idx];
395
+	buf = &pipe->bufs[i_head & p_mask];
382396 	if (off) {
383397 		if (offset == off && buf->page == page) {
384398 			/* merge with the last one */
....@@ -386,19 +400,22 @@
386400 			i->iov_offset += bytes;
387401 			goto out;
388402 		}
389
-		idx = next_idx(idx, pipe);
390
-		buf = &pipe->bufs[idx];
403
+		i_head++;
404
+		buf = &pipe->bufs[i_head & p_mask];
391405 	}
392
-	if (idx == pipe->curbuf && pipe->nrbufs)
406
+	if (pipe_full(i_head, p_tail, pipe->max_usage))
393407 		return 0;
394
-	pipe->nrbufs++;
408
+
395409 	buf->ops = &page_cache_pipe_buf_ops;
396410 	buf->flags = 0;
397
-	get_page(buf->page = page);
411
+	get_page(page);
412
+	buf->page = page;
398413 	buf->offset = offset;
399414 	buf->len = bytes;
415
+
416
+	pipe->head = i_head + 1;
400417 	i->iov_offset = offset + bytes;
401
-	i->idx = idx;
418
+	i->head = i_head;
402419 out:
403420 	i->count -= bytes;
404421 	return bytes;
....@@ -429,17 +446,19 @@
429446 }
430447 EXPORT_SYMBOL(iov_iter_fault_in_readable);
431448 
432
-void iov_iter_init(struct iov_iter *i, int direction,
449
+void iov_iter_init(struct iov_iter *i, unsigned int direction,
433450 			const struct iovec *iov, unsigned long nr_segs,
434451 			size_t count)
435452 {
453
+	WARN_ON(direction & ~(READ | WRITE));
454
+	direction &= READ | WRITE;
455
+
436456 	/* It will get better.  Eventually... */
437457 	if (uaccess_kernel()) {
438
-		direction |= ITER_KVEC;
439
-		i->type = direction;
458
+		i->type = ITER_KVEC | direction;
440459 		i->kvec = (struct kvec *)iov;
441460 	} else {
442
-		i->type = direction;
461
+		i->type = ITER_IOVEC | direction;
443462 		i->iov = iov;
444463 	}
445464 	i->nr_segs = nr_segs;
....@@ -447,20 +466,6 @@
447466 	i->count = count;
448467 }
449468 EXPORT_SYMBOL(iov_iter_init);
450
-
451
-static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
452
-{
453
-	char *from = kmap_atomic(page);
454
-	memcpy(to, from + offset, len);
455
-	kunmap_atomic(from);
456
-}
457
-
458
-static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
459
-{
460
-	char *to = kmap_atomic(page);
461
-	memcpy(to + offset, from, len);
462
-	kunmap_atomic(to);
463
-}
464469 
465470 static void memzero_page(struct page *page, size_t offset, size_t len)
466471 {
....@@ -474,24 +479,30 @@
474479 	return buf->ops == &default_pipe_buf_ops;
475480 }
476481 
477
-static inline void data_start(const struct iov_iter *i, int *idxp, size_t *offp)
482
+static inline void data_start(const struct iov_iter *i,
483
+			      unsigned int *iter_headp, size_t *offp)
478484 {
485
+	unsigned int p_mask = i->pipe->ring_size - 1;
486
+	unsigned int iter_head = i->head;
479487 	size_t off = i->iov_offset;
480
-	int idx = i->idx;
481
-	if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) {
482
-		idx = next_idx(idx, i->pipe);
488
+
489
+	if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) ||
490
+		    off == PAGE_SIZE)) {
491
+		iter_head++;
483492 		off = 0;
484493 	}
485
-	*idxp = idx;
494
+	*iter_headp = iter_head;
486495 	*offp = off;
487496 }
488497 
489498 static size_t push_pipe(struct iov_iter *i, size_t size,
490
-			int *idxp, size_t *offp)
499
+			int *iter_headp, size_t *offp)
491500 {
492501 	struct pipe_inode_info *pipe = i->pipe;
502
+	unsigned int p_tail = pipe->tail;
503
+	unsigned int p_mask = pipe->ring_size - 1;
504
+	unsigned int iter_head;
493505 	size_t off;
494
-	int idx;
495506 	ssize_t left;
496507 
497508 	if (unlikely(size > i->count))
....@@ -500,34 +511,35 @@
500511 		return 0;
501512 
502513 	left = size;
503
-	data_start(i, &idx, &off);
504
-	*idxp = idx;
514
+	data_start(i, &iter_head, &off);
515
+	*iter_headp = iter_head;
505516 	*offp = off;
506517 	if (off) {
507518 		left -= PAGE_SIZE - off;
508519 		if (left <= 0) {
509
-			pipe->bufs[idx].len += size;
520
+			pipe->bufs[iter_head & p_mask].len += size;
510521 			return size;
511522 		}
512
-		pipe->bufs[idx].len = PAGE_SIZE;
513
-		idx = next_idx(idx, pipe);
523
+		pipe->bufs[iter_head & p_mask].len = PAGE_SIZE;
524
+		iter_head++;
514525 	}
515
-	while (idx != pipe->curbuf || !pipe->nrbufs) {
526
+	while (!pipe_full(iter_head, p_tail, pipe->max_usage)) {
527
+		struct pipe_buffer *buf = &pipe->bufs[iter_head & p_mask];
516528 		struct page *page = alloc_page(GFP_USER);
517529 		if (!page)
518530 			break;
519
-		pipe->nrbufs++;
520
-		pipe->bufs[idx].ops = &default_pipe_buf_ops;
521
-		pipe->bufs[idx].flags = 0;
522
-		pipe->bufs[idx].page = page;
523
-		pipe->bufs[idx].offset = 0;
524
-		if (left <= PAGE_SIZE) {
525
-			pipe->bufs[idx].len = left;
531
+
532
+		buf->ops = &default_pipe_buf_ops;
533
+		buf->flags = 0;
534
+		buf->page = page;
535
+		buf->offset = 0;
536
+		buf->len = min_t(ssize_t, left, PAGE_SIZE);
537
+		left -= buf->len;
538
+		iter_head++;
539
+		pipe->head = iter_head;
540
+
541
+		if (left == 0)
526542 			return size;
527
-		}
528
-		pipe->bufs[idx].len = PAGE_SIZE;
529
-		left -= PAGE_SIZE;
530
-		idx = next_idx(idx, pipe);
531543 	}
532544 	return size - left;
533545 }
....@@ -536,31 +548,77 @@
536548 				struct iov_iter *i)
537549 {
538550 	struct pipe_inode_info *pipe = i->pipe;
551
+	unsigned int p_mask = pipe->ring_size - 1;
552
+	unsigned int i_head;
539553 	size_t n, off;
540
-	int idx;
541554 
542555 	if (!sanity(i))
543556 		return 0;
544557 
545
-	bytes = n = push_pipe(i, bytes, &idx, &off);
558
+	bytes = n = push_pipe(i, bytes, &i_head, &off);
546559 	if (unlikely(!n))
547560 		return 0;
548
-	for ( ; n; idx = next_idx(idx, pipe), off = 0) {
561
+	do {
549562 		size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
550
-		memcpy_to_page(pipe->bufs[idx].page, off, addr, chunk);
551
-		i->idx = idx;
563
+		memcpy_to_page(pipe->bufs[i_head & p_mask].page, off, addr, chunk);
564
+		i->head = i_head;
552565 		i->iov_offset = off + chunk;
553566 		n -= chunk;
554567 		addr += chunk;
555
-	}
568
+		off = 0;
569
+		i_head++;
570
+	} while (n);
556571 	i->count -= bytes;
572
+	return bytes;
573
+}
574
+
575
+static __wsum csum_and_memcpy(void *to, const void *from, size_t len,
576
+			      __wsum sum, size_t off)
577
+{
578
+	__wsum next = csum_partial_copy_nocheck(from, to, len);
579
+	return csum_block_add(sum, next, off);
580
+}
581
+
582
+static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes,
583
+					 struct csum_state *csstate,
584
+					 struct iov_iter *i)
585
+{
586
+	struct pipe_inode_info *pipe = i->pipe;
587
+	unsigned int p_mask = pipe->ring_size - 1;
588
+	__wsum sum = csstate->csum;
589
+	size_t off = csstate->off;
590
+	unsigned int i_head;
591
+	size_t n, r;
592
+
593
+	if (!sanity(i))
594
+		return 0;
595
+
596
+	bytes = n = push_pipe(i, bytes, &i_head, &r);
597
+	if (unlikely(!n))
598
+		return 0;
599
+	do {
600
+		size_t chunk = min_t(size_t, n, PAGE_SIZE - r);
601
+		char *p = kmap_atomic(pipe->bufs[i_head & p_mask].page);
602
+		sum = csum_and_memcpy(p + r, addr, chunk, sum, off);
603
+		kunmap_atomic(p);
604
+		i->head = i_head;
605
+		i->iov_offset = r + chunk;
606
+		n -= chunk;
607
+		off += chunk;
608
+		addr += chunk;
609
+		r = 0;
610
+		i_head++;
611
+	} while (n);
612
+	i->count -= bytes;
613
+	csstate->csum = sum;
614
+	csstate->off = off;
557615 	return bytes;
558616 }
559617 
560618 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
561619 {
562620 	const char *from = addr;
563
-	if (unlikely(i->type & ITER_PIPE))
621
+	if (unlikely(iov_iter_is_pipe(i)))
564622 		return copy_pipe_to_iter(addr, bytes, i);
565623 	if (iter_is_iovec(i))
566624 		might_fault();
....@@ -575,73 +633,75 @@
575633 }
576634 EXPORT_SYMBOL(_copy_to_iter);
577635 
578
-#ifdef CONFIG_ARCH_HAS_UACCESS_MCSAFE
579
-static int copyout_mcsafe(void __user *to, const void *from, size_t n)
636
+#ifdef CONFIG_ARCH_HAS_COPY_MC
637
+static int copyout_mc(void __user *to, const void *from, size_t n)
580638 {
581
-	if (access_ok(VERIFY_WRITE, to, n)) {
582
-		kasan_check_read(from, n);
583
-		n = copy_to_user_mcsafe((__force void *) to, from, n);
639
+	if (access_ok(to, n)) {
640
+		instrument_copy_to_user(to, from, n);
641
+		n = copy_mc_to_user((__force void *) to, from, n);
584642 	}
585643 	return n;
586644 }
587645 
588
-static unsigned long memcpy_mcsafe_to_page(struct page *page, size_t offset,
646
+static unsigned long copy_mc_to_page(struct page *page, size_t offset,
589647 		const char *from, size_t len)
590648 {
591649 	unsigned long ret;
592650 	char *to;
593651 
594652 	to = kmap_atomic(page);
595
-	ret = memcpy_mcsafe(to + offset, from, len);
653
+	ret = copy_mc_to_kernel(to + offset, from, len);
596654 	kunmap_atomic(to);
597655 
598656 	return ret;
599657 }
600658 
601
-static size_t copy_pipe_to_iter_mcsafe(const void *addr, size_t bytes,
659
+static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes,
602660 				struct iov_iter *i)
603661 {
604662 	struct pipe_inode_info *pipe = i->pipe;
663
+	unsigned int p_mask = pipe->ring_size - 1;
664
+	unsigned int i_head;
605665 	size_t n, off, xfer = 0;
606
-	int idx;
607666 
608667 	if (!sanity(i))
609668 		return 0;
610669 
611
-	bytes = n = push_pipe(i, bytes, &idx, &off);
670
+	bytes = n = push_pipe(i, bytes, &i_head, &off);
612671 	if (unlikely(!n))
613672 		return 0;
614
-	for ( ; n; idx = next_idx(idx, pipe), off = 0) {
673
+	do {
615674 		size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
616675 		unsigned long rem;
617676 
618
-		rem = memcpy_mcsafe_to_page(pipe->bufs[idx].page, off, addr,
619
-				chunk);
620
-		i->idx = idx;
677
+		rem = copy_mc_to_page(pipe->bufs[i_head & p_mask].page,
678
+					    off, addr, chunk);
679
+		i->head = i_head;
621680 		i->iov_offset = off + chunk - rem;
622681 		xfer += chunk - rem;
623682 		if (rem)
624683 			break;
625684 		n -= chunk;
626685 		addr += chunk;
627
-	}
686
+		off = 0;
687
+		i_head++;
688
+	} while (n);
628689 	i->count -= xfer;
629690 	return xfer;
630691 }
631692 
632693 /**
633
- * _copy_to_iter_mcsafe - copy to user with source-read error exception handling
694
+ * _copy_mc_to_iter - copy to iter with source memory error exception handling
634695  * @addr: source kernel address
635696  * @bytes: total transfer length
636697  * @iter: destination iterator
637698  *
638
- * The pmem driver arranges for filesystem-dax to use this facility via
639
- * dax_copy_to_iter() for protecting read/write to persistent memory.
640
- * Unless / until an architecture can guarantee identical performance
641
- * between _copy_to_iter_mcsafe() and _copy_to_iter() it would be a
642
- * performance regression to switch more users to the mcsafe version.
699
+ * The pmem driver deploys this for the dax operation
700
+ * (dax_copy_to_iter()) for dax reads (bypass page-cache and the
701
+ * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
702
+ * successfully copied.
643703  *
644
- * Otherwise, the main differences between this and typical _copy_to_iter().
704
+ * The main differences between this and typical _copy_to_iter().
645705  *
646706  * * Typical tail/residue handling after a fault retries the copy
647707  *   byte-by-byte until the fault happens again. Re-triggering machine
....@@ -652,23 +712,22 @@
652712  * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies.
653713  *   Compare to copy_to_iter() where only ITER_IOVEC attempts might return
654714  *   a short copy.
655
- *
656
- * See MCSAFE_TEST for self-test.
657715  */
658
-size_t _copy_to_iter_mcsafe(const void *addr, size_t bytes, struct iov_iter *i)
716
+size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
659717 {
660718 	const char *from = addr;
661719 	unsigned long rem, curr_addr, s_addr = (unsigned long) addr;
662720 
663
-	if (unlikely(i->type & ITER_PIPE))
664
-		return copy_pipe_to_iter_mcsafe(addr, bytes, i);
721
+	if (unlikely(iov_iter_is_pipe(i)))
722
+		return copy_mc_pipe_to_iter(addr, bytes, i);
665723 	if (iter_is_iovec(i))
666724 		might_fault();
667725 	iterate_and_advance(i, bytes, v,
668
-		copyout_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
726
+		copyout_mc(v.iov_base, (from += v.iov_len) - v.iov_len,
727
+			   v.iov_len),
669728 		({
670
-		rem = memcpy_mcsafe_to_page(v.bv_page, v.bv_offset,
671
-                               (from += v.bv_len) - v.bv_len, v.bv_len);
729
+		rem = copy_mc_to_page(v.bv_page, v.bv_offset,
730
+				      (from += v.bv_len) - v.bv_len, v.bv_len);
672731 		if (rem) {
673732 			curr_addr = (unsigned long) from;
674733 			bytes = curr_addr - s_addr - rem;
....@@ -676,8 +735,8 @@
676735 		}
677736 		}),
678737 		({
679
-		rem = memcpy_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len,
680
-				v.iov_len);
738
+		rem = copy_mc_to_kernel(v.iov_base, (from += v.iov_len)
739
+					- v.iov_len, v.iov_len);
681740 		if (rem) {
682741 			curr_addr = (unsigned long) from;
683742 			bytes = curr_addr - s_addr - rem;
....@@ -688,13 +747,13 @@
688747 
689748 	return bytes;
690749 }
691
-EXPORT_SYMBOL_GPL(_copy_to_iter_mcsafe);
692
-#endif /* CONFIG_ARCH_HAS_UACCESS_MCSAFE */
750
+EXPORT_SYMBOL_GPL(_copy_mc_to_iter);
751
+#endif /* CONFIG_ARCH_HAS_COPY_MC */
693752 
694753 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
695754 {
696755 	char *to = addr;
697
-	if (unlikely(i->type & ITER_PIPE)) {
756
+	if (unlikely(iov_iter_is_pipe(i))) {
698757 		WARN_ON(1);
699758 		return 0;
700759 	}
....@@ -714,7 +773,7 @@
714773 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
715774 {
716775 	char *to = addr;
717
-	if (unlikely(i->type & ITER_PIPE)) {
776
+	if (unlikely(iov_iter_is_pipe(i))) {
718777 		WARN_ON(1);
719778 		return false;
720779 	}
....@@ -741,7 +800,7 @@
741800 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
742801 {
743802 	char *to = addr;
744
-	if (unlikely(i->type & ITER_PIPE)) {
803
+	if (unlikely(iov_iter_is_pipe(i))) {
745804 		WARN_ON(1);
746805 		return 0;
747806 	}
....@@ -775,7 +834,7 @@
775834 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
776835 {
777836 	char *to = addr;
778
-	if (unlikely(i->type & ITER_PIPE)) {
837
+	if (unlikely(iov_iter_is_pipe(i))) {
779838 		WARN_ON(1);
780839 		return 0;
781840 	}
....@@ -796,7 +855,7 @@
796855 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
797856 {
798857 	char *to = addr;
799
-	if (unlikely(i->type & ITER_PIPE)) {
858
+	if (unlikely(iov_iter_is_pipe(i))) {
800859 		WARN_ON(1);
801860 		return false;
802861 	}
....@@ -835,7 +894,7 @@
835894 	head = compound_head(page);
836895 	v += (page - head) << PAGE_SHIFT;
837896 
838
-	if (likely(n <= v && v <= (PAGE_SIZE << compound_order(head))))
897
+	if (likely(n <= v && v <= (page_size(head))))
839898 		return true;
840899 	WARN_ON(1);
841900 	return false;
....@@ -851,7 +910,12 @@
851910 		size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
852911 		kunmap_atomic(kaddr);
853912 		return wanted;
854
-	} else if (likely(!(i->type & ITER_PIPE)))
913
+	} else if (unlikely(iov_iter_is_discard(i))) {
914
+		if (unlikely(i->count < bytes))
915
+			bytes = i->count;
916
+		i->count -= bytes;
917
+		return bytes;
918
+	} else if (likely(!iov_iter_is_pipe(i)))
855919 		return copy_page_to_iter_iovec(page, offset, bytes, i);
856920 	else
857921 		return copy_page_to_iter_pipe(page, offset, bytes, i);
....@@ -863,7 +927,7 @@
863927 {
864928 	if (unlikely(!page_copy_sane(page, offset, bytes)))
865929 		return 0;
866
-	if (unlikely(i->type & ITER_PIPE)) {
930
+	if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
867931 		WARN_ON(1);
868932 		return 0;
869933 	}
....@@ -880,30 +944,33 @@
880944 static size_t pipe_zero(size_t bytes, struct iov_iter *i)
881945 {
882946 	struct pipe_inode_info *pipe = i->pipe;
947
+	unsigned int p_mask = pipe->ring_size - 1;
948
+	unsigned int i_head;
883949 	size_t n, off;
884
-	int idx;
885950 
886951 	if (!sanity(i))
887952 		return 0;
888953 
889
-	bytes = n = push_pipe(i, bytes, &idx, &off);
954
+	bytes = n = push_pipe(i, bytes, &i_head, &off);
890955 	if (unlikely(!n))
891956 		return 0;
892957 
893
-	for ( ; n; idx = next_idx(idx, pipe), off = 0) {
958
+	do {
894959 		size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
895
-		memzero_page(pipe->bufs[idx].page, off, chunk);
896
-		i->idx = idx;
960
+		memzero_page(pipe->bufs[i_head & p_mask].page, off, chunk);
961
+		i->head = i_head;
897962 		i->iov_offset = off + chunk;
898963 		n -= chunk;
899
-	}
964
+		off = 0;
965
+		i_head++;
966
+	} while (n);
900967 	i->count -= bytes;
901968 	return bytes;
902969 }
903970 
904971 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
905972 {
906
-	if (unlikely(i->type & ITER_PIPE))
973
+	if (unlikely(iov_iter_is_pipe(i)))
907974 		return pipe_zero(bytes, i);
908975 	iterate_and_advance(i, bytes, v,
909976 		clear_user(v.iov_base, v.iov_len),
....@@ -923,7 +990,7 @@
923990 		kunmap_atomic(kaddr);
924991 		return 0;
925992 	}
926
-	if (unlikely(i->type & ITER_PIPE)) {
993
+	if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
927994 		kunmap_atomic(kaddr);
928995 		WARN_ON(1);
929996 		return 0;
....@@ -942,20 +1009,26 @@
9421009 static inline void pipe_truncate(struct iov_iter *i)
9431010 {
9441011 	struct pipe_inode_info *pipe = i->pipe;
945
-	if (pipe->nrbufs) {
1012
+	unsigned int p_tail = pipe->tail;
1013
+	unsigned int p_head = pipe->head;
1014
+	unsigned int p_mask = pipe->ring_size - 1;
1015
+
1016
+	if (!pipe_empty(p_head, p_tail)) {
1017
+		struct pipe_buffer *buf;
1018
+		unsigned int i_head = i->head;
9461019 		size_t off = i->iov_offset;
947
-		int idx = i->idx;
948
-		int nrbufs = (idx - pipe->curbuf) & (pipe->buffers - 1);
1020
+
9491021 		if (off) {
950
-			pipe->bufs[idx].len = off - pipe->bufs[idx].offset;
951
-			idx = next_idx(idx, pipe);
952
-			nrbufs++;
1022
+			buf = &pipe->bufs[i_head & p_mask];
1023
+			buf->len = off - buf->offset;
1024
+			i_head++;
9531025 		}
954
-		while (pipe->nrbufs > nrbufs) {
955
-			pipe_buf_release(pipe, &pipe->bufs[idx]);
956
-			idx = next_idx(idx, pipe);
957
-			pipe->nrbufs--;
1026
+		while (p_head != i_head) {
1027
+			p_head--;
1028
+			pipe_buf_release(pipe, &pipe->bufs[p_head & p_mask]);
9581029 		}
1030
+
1031
+		pipe->head = p_head;
9591032 	}
9601033 }
9611034 
....@@ -966,18 +1039,20 @@
9661039 		size = i->count;
9671040 	if (size) {
9681041 		struct pipe_buffer *buf;
1042
+		unsigned int p_mask = pipe->ring_size - 1;
1043
+		unsigned int i_head = i->head;
9691044 		size_t off = i->iov_offset, left = size;
970
-		int idx = i->idx;
1045
+
9711046 		if (off) /* make it relative to the beginning of buffer */
972
-			left += off - pipe->bufs[idx].offset;
1047
+			left += off - pipe->bufs[i_head & p_mask].offset;
9731048 		while (1) {
974
-			buf = &pipe->bufs[idx];
1049
+			buf = &pipe->bufs[i_head & p_mask];
9751050 			if (left <= buf->len)
9761051 				break;
9771052 			left -= buf->len;
978
-			idx = next_idx(idx, pipe);
1053
+			i_head++;
9791054 		}
980
-		i->idx = idx;
1055
+		i->head = i_head;
9811056 		i->iov_offset = buf->offset + left;
9821057 	}
9831058 	i->count -= size;
....@@ -987,8 +1062,12 @@
9871062 
9881063 void iov_iter_advance(struct iov_iter *i, size_t size)
9891064 {
990
-	if (unlikely(i->type & ITER_PIPE)) {
1065
+	if (unlikely(iov_iter_is_pipe(i))) {
9911066 		pipe_advance(i, size);
1067
+		return;
1068
+	}
1069
+	if (unlikely(iov_iter_is_discard(i))) {
1070
+		i->count -= size;
9921071 		return;
9931072 	}
9941073 	iterate_and_advance(i, size, v, 0, 0, 0)
....@@ -1002,36 +1081,40 @@
10021081 	if (WARN_ON(unroll > MAX_RW_COUNT))
10031082 		return;
10041083 	i->count += unroll;
1005
-	if (unlikely(i->type & ITER_PIPE)) {
1084
+	if (unlikely(iov_iter_is_pipe(i))) {
10061085 		struct pipe_inode_info *pipe = i->pipe;
1007
-		int idx = i->idx;
1086
+		unsigned int p_mask = pipe->ring_size - 1;
1087
+		unsigned int i_head = i->head;
10081088 		size_t off = i->iov_offset;
10091089 		while (1) {
1010
-			size_t n = off - pipe->bufs[idx].offset;
1090
+			struct pipe_buffer *b = &pipe->bufs[i_head & p_mask];
1091
+			size_t n = off - b->offset;
10111092 			if (unroll < n) {
10121093 				off -= unroll;
10131094 				break;
10141095 			}
10151096 			unroll -= n;
1016
-			if (!unroll && idx == i->start_idx) {
1097
+			if (!unroll && i_head == i->start_head) {
10171098 				off = 0;
10181099 				break;
10191100 			}
1020
-			if (!idx--)
1021
-				idx = pipe->buffers - 1;
1022
-			off = pipe->bufs[idx].offset + pipe->bufs[idx].len;
1101
+			i_head--;
1102
+			b = &pipe->bufs[i_head & p_mask];
1103
+			off = b->offset + b->len;
10231104 		}
10241105 		i->iov_offset = off;
1025
-		i->idx = idx;
1106
+		i->head = i_head;
10261107 		pipe_truncate(i);
10271108 		return;
10281109 	}
1110
+	if (unlikely(iov_iter_is_discard(i)))
1111
+		return;
10291112 	if (unroll <= i->iov_offset) {
10301113 		i->iov_offset -= unroll;
10311114 		return;
10321115 	}
10331116 	unroll -= i->iov_offset;
1034
-	if (i->type & ITER_BVEC) {
1117
+	if (iov_iter_is_bvec(i)) {
10351118 		const struct bio_vec *bvec = i->bvec;
10361119 		while (1) {
10371120 			size_t n = (--bvec)->bv_len;
....@@ -1064,23 +1147,25 @@
10641147  */
10651148 size_t iov_iter_single_seg_count(const struct iov_iter *i)
10661149 {
1067
-	if (unlikely(i->type & ITER_PIPE))
1150
+	if (unlikely(iov_iter_is_pipe(i)))
10681151 		return i->count;	// it is a silly place, anyway
10691152 	if (i->nr_segs == 1)
10701153 		return i->count;
1071
-	else if (i->type & ITER_BVEC)
1154
+	if (unlikely(iov_iter_is_discard(i)))
1155
+		return i->count;
1156
+	else if (iov_iter_is_bvec(i))
10721157 		return min(i->count, i->bvec->bv_len - i->iov_offset);
10731158 	else
10741159 		return min(i->count, i->iov->iov_len - i->iov_offset);
10751160 }
10761161 EXPORT_SYMBOL(iov_iter_single_seg_count);
10771162 
1078
-void iov_iter_kvec(struct iov_iter *i, int direction,
1163
+void iov_iter_kvec(struct iov_iter *i, unsigned int direction,
10791164 			const struct kvec *kvec, unsigned long nr_segs,
10801165 			size_t count)
10811166 {
1082
-	BUG_ON(!(direction & ITER_KVEC));
1083
-	i->type = direction;
1167
+	WARN_ON(direction & ~(READ | WRITE));
1168
+	i->type = ITER_KVEC | (direction & (READ | WRITE));
10841169 	i->kvec = kvec;
10851170 	i->nr_segs = nr_segs;
10861171 	i->iov_offset = 0;
....@@ -1088,12 +1173,12 @@
10881173 }
10891174 EXPORT_SYMBOL(iov_iter_kvec);
10901175 
1091
-void iov_iter_bvec(struct iov_iter *i, int direction,
1176
+void iov_iter_bvec(struct iov_iter *i, unsigned int direction,
10921177 			const struct bio_vec *bvec, unsigned long nr_segs,
10931178 			size_t count)
10941179 {
1095
-	BUG_ON(!(direction & ITER_BVEC));
1096
-	i->type = direction;
1180
+	WARN_ON(direction & ~(READ | WRITE));
1181
+	i->type = ITER_BVEC | (direction & (READ | WRITE));
10971182 	i->bvec = bvec;
10981183 	i->nr_segs = nr_segs;
10991184 	i->iov_offset = 0;
....@@ -1101,28 +1186,48 @@
11011186 }
11021187 EXPORT_SYMBOL(iov_iter_bvec);
11031188 
1104
-void iov_iter_pipe(struct iov_iter *i, int direction,
1189
+void iov_iter_pipe(struct iov_iter *i, unsigned int direction,
11051190 			struct pipe_inode_info *pipe,
11061191 			size_t count)
11071192 {
1108
-	BUG_ON(direction != ITER_PIPE);
1109
-	WARN_ON(pipe->nrbufs == pipe->buffers);
1110
-	i->type = direction;
1193
+	BUG_ON(direction != READ);
1194
+	WARN_ON(pipe_full(pipe->head, pipe->tail, pipe->ring_size));
1195
+	i->type = ITER_PIPE | READ;
11111196 	i->pipe = pipe;
1112
-	i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
1197
+	i->head = pipe->head;
11131198 	i->iov_offset = 0;
11141199 	i->count = count;
1115
-	i->start_idx = i->idx;
1200
+	i->start_head = i->head;
11161201 }
11171202 EXPORT_SYMBOL(iov_iter_pipe);
1203
+
1204
+/**
1205
+ * iov_iter_discard - Initialise an I/O iterator that discards data
1206
+ * @i: The iterator to initialise.
1207
+ * @direction: The direction of the transfer.
1208
+ * @count: The size of the I/O buffer in bytes.
1209
+ *
1210
+ * Set up an I/O iterator that just discards everything that's written to it.
1211
+ * It's only available as a READ iterator.
1212
+ */
1213
+void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count)
1214
+{
1215
+	BUG_ON(direction != READ);
1216
+	i->type = ITER_DISCARD | READ;
1217
+	i->count = count;
1218
+	i->iov_offset = 0;
1219
+}
1220
+EXPORT_SYMBOL(iov_iter_discard);
11181221 
11191222 unsigned long iov_iter_alignment(const struct iov_iter *i)
11201223 {
11211224 	unsigned long res = 0;
11221225 	size_t size = i->count;
11231226 
1124
-	if (unlikely(i->type & ITER_PIPE)) {
1125
-		if (size && i->iov_offset && allocated(&i->pipe->bufs[i->idx]))
1227
+	if (unlikely(iov_iter_is_pipe(i))) {
1228
+		unsigned int p_mask = i->pipe->ring_size - 1;
1229
+
1230
+		if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask]))
11261231 			return size | i->iov_offset;
11271232 		return size;
11281233 	}
....@@ -1140,7 +1245,7 @@
11401245 	unsigned long res = 0;
11411246 	size_t size = i->count;
11421247 
1143
-	if (unlikely(i->type & ITER_PIPE)) {
1248
+	if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
11441249 		WARN_ON(1);
11451250 		return ~0U;
11461251 	}
....@@ -1160,19 +1265,20 @@
11601265 static inline ssize_t __pipe_get_pages(struct iov_iter *i,
11611266 				size_t maxsize,
11621267 				struct page **pages,
1163
-				int idx,
1268
+				int iter_head,
11641269 				size_t *start)
11651270 {
11661271 	struct pipe_inode_info *pipe = i->pipe;
1167
-	ssize_t n = push_pipe(i, maxsize, &idx, start);
1272
+	unsigned int p_mask = pipe->ring_size - 1;
1273
+	ssize_t n = push_pipe(i, maxsize, &iter_head, start);
11681274 	if (!n)
11691275 		return -EFAULT;
11701276 
11711277 	maxsize = n;
11721278 	n += *start;
11731279 	while (n > 0) {
1174
-		get_page(*pages++ = pipe->bufs[idx].page);
1175
-		idx = next_idx(idx, pipe);
1280
+		get_page(*pages++ = pipe->bufs[iter_head & p_mask].page);
1281
+		iter_head++;
11761282 		n -= PAGE_SIZE;
11771283 	}
11781284 
....@@ -1183,9 +1289,8 @@
11831289 		   struct page **pages, size_t maxsize, unsigned maxpages,
11841290 		   size_t *start)
11851291 {
1186
-	unsigned npages;
1292
+	unsigned int iter_head, npages;
11871293 	size_t capacity;
1188
-	int idx;
11891294 
11901295 	if (!maxsize)
11911296 		return 0;
....@@ -1193,12 +1298,12 @@
11931298 	if (!sanity(i))
11941299 		return -EFAULT;
11951300 
1196
-	data_start(i, &idx, start);
1197
-	/* some of this one + all after this one */
1198
-	npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
1199
-	capacity = min(npages,maxpages) * PAGE_SIZE - *start;
1301
+	data_start(i, &iter_head, start);
1302
+	/* Amount of free space: some of this one + all after this one */
1303
+	npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);
1304
+	capacity = min(npages, maxpages) * PAGE_SIZE - *start;
12001305 
1201
-	return __pipe_get_pages(i, min(maxsize, capacity), pages, idx, start);
1306
+	return __pipe_get_pages(i, min(maxsize, capacity), pages, iter_head, start);
12021307 }
12031308 
12041309 ssize_t iov_iter_get_pages(struct iov_iter *i,
....@@ -1208,8 +1313,11 @@
12081313 	if (maxsize > i->count)
12091314 		maxsize = i->count;
12101315 
1211
-	if (unlikely(i->type & ITER_PIPE))
1316
+	if (unlikely(iov_iter_is_pipe(i)))
12121317 		return pipe_get_pages(i, pages, maxsize, maxpages, start);
1318
+	if (unlikely(iov_iter_is_discard(i)))
1319
+		return -EFAULT;
1320
+
12131321 	iterate_all_kinds(i, maxsize, v, ({
12141322 		unsigned long addr = (unsigned long)v.iov_base;
12151323 		size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
....@@ -1220,8 +1328,10 @@
12201328 			len = maxpages * PAGE_SIZE;
12211329 		addr &= ~(PAGE_SIZE - 1);
12221330 		n = DIV_ROUND_UP(len, PAGE_SIZE);
1223
-		res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
1224
-		if (unlikely(res < 0))
1331
+		res = get_user_pages_fast(addr, n,
1332
+				iov_iter_rw(i) != WRITE ?  FOLL_WRITE : 0,
1333
+				pages);
1334
+		if (unlikely(res <= 0))
12251335 			return res;
12261336 		return (res == n ? len : res * PAGE_SIZE) - *start;
12271337 	0;}),({
....@@ -1247,9 +1357,8 @@
12471357 		   size_t *start)
12481358 {
12491359 	struct page **p;
1360
+	unsigned int iter_head, npages;
12501361 	ssize_t n;
1251
-	int idx;
1252
-	int npages;
12531362 
12541363 	if (!maxsize)
12551364 		return 0;
....@@ -1257,9 +1366,9 @@
12571366 	if (!sanity(i))
12581367 		return -EFAULT;
12591368 
1260
-	data_start(i, &idx, start);
1261
-	/* some of this one + all after this one */
1262
-	npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
1369
+	data_start(i, &iter_head, start);
1370
+	/* Amount of free space: some of this one + all after this one */
1371
+	npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);
12631372 	n = npages * PAGE_SIZE - *start;
12641373 	if (maxsize > n)
12651374 		maxsize = n;
....@@ -1268,7 +1377,7 @@
12681377 	p = get_pages_array(npages);
12691378 	if (!p)
12701379 		return -ENOMEM;
1271
-	n = __pipe_get_pages(i, maxsize, p, idx, start);
1380
+	n = __pipe_get_pages(i, maxsize, p, iter_head, start);
12721381 	if (n > 0)
12731382 		*pages = p;
12741383 	else
....@@ -1285,8 +1394,11 @@
12851394 	if (maxsize > i->count)
12861395 		maxsize = i->count;
12871396 
1288
-	if (unlikely(i->type & ITER_PIPE))
1397
+	if (unlikely(iov_iter_is_pipe(i)))
12891398 		return pipe_get_pages_alloc(i, pages, maxsize, start);
1399
+	if (unlikely(iov_iter_is_discard(i)))
1400
+		return -EFAULT;
1401
+
12901402 	iterate_all_kinds(i, maxsize, v, ({
12911403 		unsigned long addr = (unsigned long)v.iov_base;
12921404 		size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
....@@ -1298,9 +1410,11 @@
12981410 		p = get_pages_array(n);
12991411 		if (!p)
13001412 			return -ENOMEM;
1301
-		res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
1302
-		if (unlikely(res < 0)) {
1413
+		res = get_user_pages_fast(addr, n,
1414
+				iov_iter_rw(i) != WRITE ?  FOLL_WRITE : 0, p);
1415
+		if (unlikely(res <= 0)) {
13031416 			kvfree(p);
1417
+			*pages = NULL;
13041418 			return res;
13051419 		}
13061420 		*pages = p;
....@@ -1328,33 +1442,30 @@
13281442 	__wsum sum, next;
13291443 	size_t off = 0;
13301444 	sum = *csum;
1331
-	if (unlikely(i->type & ITER_PIPE)) {
1445
+	if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
13321446 		WARN_ON(1);
13331447 		return 0;
13341448 	}
13351449 	iterate_and_advance(i, bytes, v, ({
1336
-		int err = 0;
13371450 		next = csum_and_copy_from_user(v.iov_base,
13381451 					       (to += v.iov_len) - v.iov_len,
1339
-					       v.iov_len, 0, &err);
1340
-		if (!err) {
1452
+					       v.iov_len);
1453
+		if (next) {
13411454 			sum = csum_block_add(sum, next, off);
13421455 			off += v.iov_len;
13431456 		}
1344
-		err ? v.iov_len : 0;
1457
+		next ? 0 : v.iov_len;
13451458 	}), ({
13461459 		char *p = kmap_atomic(v.bv_page);
1347
-		next = csum_partial_copy_nocheck(p + v.bv_offset,
1348
-						 (to += v.bv_len) - v.bv_len,
1349
-						 v.bv_len, 0);
1460
+		sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1461
+				      p + v.bv_offset, v.bv_len,
1462
+				      sum, off);
13501463 		kunmap_atomic(p);
1351
-		sum = csum_block_add(sum, next, off);
13521464 		off += v.bv_len;
13531465 	}),({
1354
-		next = csum_partial_copy_nocheck(v.iov_base,
1355
-						 (to += v.iov_len) - v.iov_len,
1356
-						 v.iov_len, 0);
1357
-		sum = csum_block_add(sum, next, off);
1466
+		sum = csum_and_memcpy((to += v.iov_len) - v.iov_len,
1467
+				      v.iov_base, v.iov_len,
1468
+				      sum, off);
13581469 		off += v.iov_len;
13591470 	})
13601471 	)
....@@ -1370,35 +1481,32 @@
13701481 	__wsum sum, next;
13711482 	size_t off = 0;
13721483 	sum = *csum;
1373
-	if (unlikely(i->type & ITER_PIPE)) {
1484
+	if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
13741485 		WARN_ON(1);
13751486 		return false;
13761487 	}
13771488 	if (unlikely(i->count < bytes))
13781489 		return false;
13791490 	iterate_all_kinds(i, bytes, v, ({
1380
-		int err = 0;
13811491 		next = csum_and_copy_from_user(v.iov_base,
13821492 					       (to += v.iov_len) - v.iov_len,
1383
-					       v.iov_len, 0, &err);
1384
-		if (err)
1493
+					       v.iov_len);
1494
+		if (!next)
13851495 			return false;
13861496 		sum = csum_block_add(sum, next, off);
13871497 		off += v.iov_len;
13881498 		0;
13891499 	}), ({
13901500 		char *p = kmap_atomic(v.bv_page);
1391
-		next = csum_partial_copy_nocheck(p + v.bv_offset,
1392
-						 (to += v.bv_len) - v.bv_len,
1393
-						 v.bv_len, 0);
1501
+		sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1502
+				      p + v.bv_offset, v.bv_len,
1503
+				      sum, off);
13941504 		kunmap_atomic(p);
1395
-		sum = csum_block_add(sum, next, off);
13961505 		off += v.bv_len;
13971506 	}),({
1398
-		next = csum_partial_copy_nocheck(v.iov_base,
1399
-						 (to += v.iov_len) - v.iov_len,
1400
-						 v.iov_len, 0);
1401
-		sum = csum_block_add(sum, next, off);
1507
+		sum = csum_and_memcpy((to += v.iov_len) - v.iov_len,
1508
+				      v.iov_base, v.iov_len,
1509
+				      sum, off);
14021510 		off += v.iov_len;
14031511 	})
14041512 	)
....@@ -1408,47 +1516,70 @@
14081516 }
14091517 EXPORT_SYMBOL(csum_and_copy_from_iter_full);
14101518 
1411
-size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
1519
+size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate,
14121520 			     struct iov_iter *i)
14131521 {
1522
+	struct csum_state *csstate = _csstate;
14141523 	const char *from = addr;
14151524 	__wsum sum, next;
1416
-	size_t off = 0;
1417
-	sum = *csum;
1418
-	if (unlikely(i->type & ITER_PIPE)) {
1525
+	size_t off;
1526
+
1527
+	if (unlikely(iov_iter_is_pipe(i)))
1528
+		return csum_and_copy_to_pipe_iter(addr, bytes, _csstate, i);
1529
+
1530
+	sum = csstate->csum;
1531
+	off = csstate->off;
1532
+	if (unlikely(iov_iter_is_discard(i))) {
14191533 		WARN_ON(1);	/* for now */
14201534 		return 0;
14211535 	}
14221536 	iterate_and_advance(i, bytes, v, ({
1423
-		int err = 0;
14241537 		next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
14251538 					     v.iov_base,
1426
-					     v.iov_len, 0, &err);
1427
-		if (!err) {
1539
+					     v.iov_len);
1540
+		if (next) {
14281541 			sum = csum_block_add(sum, next, off);
14291542 			off += v.iov_len;
14301543 		}
1431
-		err ? v.iov_len : 0;
1544
+		next ? 0 : v.iov_len;
14321545 	}), ({
14331546 		char *p = kmap_atomic(v.bv_page);
1434
-		next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
1435
-						 p + v.bv_offset,
1436
-						 v.bv_len, 0);
1547
+		sum = csum_and_memcpy(p + v.bv_offset,
1548
+				      (from += v.bv_len) - v.bv_len,
1549
+				      v.bv_len, sum, off);
14371550 		kunmap_atomic(p);
1438
-		sum = csum_block_add(sum, next, off);
14391551 		off += v.bv_len;
14401552 	}),({
1441
-		next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
1442
-						 v.iov_base,
1443
-						 v.iov_len, 0);
1444
-		sum = csum_block_add(sum, next, off);
1553
+		sum = csum_and_memcpy(v.iov_base,
1554
+				     (from += v.iov_len) - v.iov_len,
1555
+				     v.iov_len, sum, off);
14451556 		off += v.iov_len;
14461557 	})
14471558 	)
1448
-	*csum = sum;
1559
+	csstate->csum = sum;
1560
+	csstate->off = off;
14491561 	return bytes;
14501562 }
14511563 EXPORT_SYMBOL(csum_and_copy_to_iter);
1564
+
1565
+size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
1566
+		struct iov_iter *i)
1567
+{
1568
+#ifdef CONFIG_CRYPTO_HASH
1569
+	struct ahash_request *hash = hashp;
1570
+	struct scatterlist sg;
1571
+	size_t copied;
1572
+
1573
+	copied = copy_to_iter(addr, bytes, i);
1574
+	sg_init_one(&sg, addr, copied);
1575
+	ahash_request_set_crypt(hash, &sg, NULL, copied);
1576
+	crypto_ahash_update(hash);
1577
+	return copied;
1578
+#else
1579
+	return 0;
1580
+#endif
1581
+}
1582
+EXPORT_SYMBOL(hash_and_copy_to_iter);
14521583 
14531584 int iov_iter_npages(const struct iov_iter *i, int maxpages)
14541585 {
....@@ -1457,18 +1588,20 @@
14571588 
14581589 	if (!size)
14591590 		return 0;
1591
+	if (unlikely(iov_iter_is_discard(i)))
1592
+		return 0;
14601593 
1461
-	if (unlikely(i->type & ITER_PIPE)) {
1594
+	if (unlikely(iov_iter_is_pipe(i))) {
14621595 		struct pipe_inode_info *pipe = i->pipe;
1596
+		unsigned int iter_head;
14631597 		size_t off;
1464
-		int idx;
14651598 
14661599 		if (!sanity(i))
14671600 			return 0;
14681601 
1469
-		data_start(i, &idx, &off);
1602
+		data_start(i, &iter_head, &off);
14701603 		/* some of this one + all after this one */
1471
-		npages = ((pipe->curbuf - idx - 1) & (pipe->buffers - 1)) + 1;
1604
+		npages = pipe_space_for_user(iter_head, pipe->tail, pipe);
14721605 		if (npages >= maxpages)
14731606 			return maxpages;
14741607 	} else iterate_all_kinds(i, size, v, ({
....@@ -1496,11 +1629,13 @@
14961629 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
14971630 {
14981631 	*new = *old;
1499
-	if (unlikely(new->type & ITER_PIPE)) {
1632
+	if (unlikely(iov_iter_is_pipe(new))) {
15001633 		WARN_ON(1);
15011634 		return NULL;
15021635 	}
1503
-	if (new->type & ITER_BVEC)
1636
+	if (unlikely(iov_iter_is_discard(new)))
1637
+		return NULL;
1638
+	if (iov_iter_is_bvec(new))
15041639 		return new->bvec = kmemdup(new->bvec,
15051640 				    new->nr_segs * sizeof(struct bio_vec),
15061641 				    flags);
....@@ -1512,16 +1647,145 @@
15121647 }
15131648 EXPORT_SYMBOL(dup_iter);
15141649 
1650
+static int copy_compat_iovec_from_user(struct iovec *iov,
1651
+		const struct iovec __user *uvec, unsigned long nr_segs)
1652
+{
1653
+	const struct compat_iovec __user *uiov =
1654
+		(const struct compat_iovec __user *)uvec;
1655
+	int ret = -EFAULT, i;
1656
+
1657
+	if (!user_access_begin(uiov, nr_segs * sizeof(*uiov)))
1658
+		return -EFAULT;
1659
+
1660
+	for (i = 0; i < nr_segs; i++) {
1661
+		compat_uptr_t buf;
1662
+		compat_ssize_t len;
1663
+
1664
+		unsafe_get_user(len, &uiov[i].iov_len, uaccess_end);
1665
+		unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end);
1666
+
1667
+		/* check for compat_size_t not fitting in compat_ssize_t .. */
1668
+		if (len < 0) {
1669
+			ret = -EINVAL;
1670
+			goto uaccess_end;
1671
+		}
1672
+		iov[i].iov_base = compat_ptr(buf);
1673
+		iov[i].iov_len = len;
1674
+	}
1675
+
1676
+	ret = 0;
1677
+uaccess_end:
1678
+	user_access_end();
1679
+	return ret;
1680
+}
1681
+
1682
+static int copy_iovec_from_user(struct iovec *iov,
1683
+		const struct iovec __user *uvec, unsigned long nr_segs)
1684
+{
1685
+	unsigned long seg;
1686
+
1687
+	if (copy_from_user(iov, uvec, nr_segs * sizeof(*uvec)))
1688
+		return -EFAULT;
1689
+	for (seg = 0; seg < nr_segs; seg++) {
1690
+		if ((ssize_t)iov[seg].iov_len < 0)
1691
+			return -EINVAL;
1692
+	}
1693
+
1694
+	return 0;
1695
+}
1696
+
1697
+struct iovec *iovec_from_user(const struct iovec __user *uvec,
1698
+		unsigned long nr_segs, unsigned long fast_segs,
1699
+		struct iovec *fast_iov, bool compat)
1700
+{
1701
+	struct iovec *iov = fast_iov;
1702
+	int ret;
1703
+
1704
+	/*
1705
+	 * SuS says "The readv() function *may* fail if the iovcnt argument was
1706
+	 * less than or equal to 0, or greater than {IOV_MAX}.  Linux has
1707
+	 * traditionally returned zero for zero segments, so...
1708
+	 */
1709
+	if (nr_segs == 0)
1710
+		return iov;
1711
+	if (nr_segs > UIO_MAXIOV)
1712
+		return ERR_PTR(-EINVAL);
1713
+	if (nr_segs > fast_segs) {
1714
+		iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
1715
+		if (!iov)
1716
+			return ERR_PTR(-ENOMEM);
1717
+	}
1718
+
1719
+	if (compat)
1720
+		ret = copy_compat_iovec_from_user(iov, uvec, nr_segs);
1721
+	else
1722
+		ret = copy_iovec_from_user(iov, uvec, nr_segs);
1723
+	if (ret) {
1724
+		if (iov != fast_iov)
1725
+			kfree(iov);
1726
+		return ERR_PTR(ret);
1727
+	}
1728
+
1729
+	return iov;
1730
+}
1731
+
1732
+ssize_t __import_iovec(int type, const struct iovec __user *uvec,
1733
+		 unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
1734
+		 struct iov_iter *i, bool compat)
1735
+{
1736
+	ssize_t total_len = 0;
1737
+	unsigned long seg;
1738
+	struct iovec *iov;
1739
+
1740
+	iov = iovec_from_user(uvec, nr_segs, fast_segs, *iovp, compat);
1741
+	if (IS_ERR(iov)) {
1742
+		*iovp = NULL;
1743
+		return PTR_ERR(iov);
1744
+	}
1745
+
1746
+	/*
1747
+	 * According to the Single Unix Specification we should return EINVAL if
1748
+	 * an element length is < 0 when cast to ssize_t or if the total length
1749
+	 * would overflow the ssize_t return value of the system call.
1750
+	 *
1751
+	 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
1752
+	 * overflow case.
1753
+	 */
1754
+	for (seg = 0; seg < nr_segs; seg++) {
1755
+		ssize_t len = (ssize_t)iov[seg].iov_len;
1756
+
1757
+		if (!access_ok(iov[seg].iov_base, len)) {
1758
+			if (iov != *iovp)
1759
+				kfree(iov);
1760
+			*iovp = NULL;
1761
+			return -EFAULT;
1762
+		}
1763
+
1764
+		if (len > MAX_RW_COUNT - total_len) {
1765
+			len = MAX_RW_COUNT - total_len;
1766
+			iov[seg].iov_len = len;
1767
+		}
1768
+		total_len += len;
1769
+	}
1770
+
1771
+	iov_iter_init(i, type, iov, nr_segs, total_len);
1772
+	if (iov == *iovp)
1773
+		*iovp = NULL;
1774
+	else
1775
+		*iovp = iov;
1776
+	return total_len;
1777
+}
1778
+
15151779 /**
15161780  * import_iovec() - Copy an array of &struct iovec from userspace
15171781  *     into the kernel, check that it is valid, and initialize a new
15181782  *     &struct iov_iter iterator to access it.
15191783  *
15201784  * @type: One of %READ or %WRITE.
1521
- * @uvector: Pointer to the userspace array.
1785
+ * @uvec: Pointer to the userspace array.
15221786  * @nr_segs: Number of elements in userspace array.
15231787  * @fast_segs: Number of elements in @iov.
1524
- * @iov: (input and output parameter) Pointer to pointer to (usually small
1788
+ * @iovp: (input and output parameter) Pointer to pointer to (usually small
15251789  *     on-stack) kernel array.
15261790  * @i: Pointer to iterator that will be initialized on success.
15271791  *
....@@ -1532,57 +1796,23 @@
15321796  * on-stack array was used or not (and regardless of whether this function
15331797  * returns an error or not).
15341798  *
1535
- * Return: 0 on success or negative error code on error.
1799
+ * Return: Negative error code on error, bytes imported on success
15361800  */
1537
-int import_iovec(int type, const struct iovec __user * uvector,
1801
+ssize_t import_iovec(int type, const struct iovec __user *uvec,
15381802 		 unsigned nr_segs, unsigned fast_segs,
1539
-		 struct iovec **iov, struct iov_iter *i)
1803
+		 struct iovec **iovp, struct iov_iter *i)
15401804 {
1541
-	ssize_t n;
1542
-	struct iovec *p;
1543
-	n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
1544
-				  *iov, &p);
1545
-	if (n < 0) {
1546
-		if (p != *iov)
1547
-			kfree(p);
1548
-		*iov = NULL;
1549
-		return n;
1550
-	}
1551
-	iov_iter_init(i, type, p, nr_segs, n);
1552
-	*iov = p == *iov ? NULL : p;
1553
-	return 0;
1805
+	return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i,
1806
+			      in_compat_syscall());
15541807 }
15551808 EXPORT_SYMBOL(import_iovec);
1556
-
1557
-#ifdef CONFIG_COMPAT
1558
-#include <linux/compat.h>
1559
-
1560
-int compat_import_iovec(int type, const struct compat_iovec __user * uvector,
1561
-		 unsigned nr_segs, unsigned fast_segs,
1562
-		 struct iovec **iov, struct iov_iter *i)
1563
-{
1564
-	ssize_t n;
1565
-	struct iovec *p;
1566
-	n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
1567
-				  *iov, &p);
1568
-	if (n < 0) {
1569
-		if (p != *iov)
1570
-			kfree(p);
1571
-		*iov = NULL;
1572
-		return n;
1573
-	}
1574
-	iov_iter_init(i, type, p, nr_segs, n);
1575
-	*iov = p == *iov ? NULL : p;
1576
-	return 0;
1577
-}
1578
-#endif
15791809 
15801810 int import_single_range(int rw, void __user *buf, size_t len,
15811811 		 struct iovec *iov, struct iov_iter *i)
15821812 {
15831813 	if (len > MAX_RW_COUNT)
15841814 		len = MAX_RW_COUNT;
1585
-	if (unlikely(!access_ok(!rw, buf, len)))
1815
+	if (unlikely(!access_ok(buf, len)))
15861816 		return -EFAULT;
15871817 
15881818 	iov->iov_base = buf;
....@@ -1592,24 +1822,38 @@
15921822 }
15931823 EXPORT_SYMBOL(import_single_range);
15941824 
1595
-int iov_iter_for_each_range(struct iov_iter *i, size_t bytes,
1596
-			    int (*f)(struct kvec *vec, void *context),
1597
-			    void *context)
1825
+/**
1826
+ * iov_iter_restore() - Restore a &struct iov_iter to the same state as when
1827
+ *     iov_iter_save_state() was called.
1828
+ *
1829
+ * @i: &struct iov_iter to restore
1830
+ * @state: state to restore from
1831
+ *
1832
+ * Used after iov_iter_save_state() to bring restore @i, if operations may
1833
+ * have advanced it.
1834
+ *
1835
+ * Note: only works on ITER_IOVEC, ITER_BVEC, and ITER_KVEC
1836
+ */
1837
+void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state)
15981838 {
1599
-	struct kvec w;
1600
-	int err = -EINVAL;
1601
-	if (!bytes)
1602
-		return 0;
1603
-
1604
-	iterate_all_kinds(i, bytes, v, -EINVAL, ({
1605
-		w.iov_base = kmap(v.bv_page) + v.bv_offset;
1606
-		w.iov_len = v.bv_len;
1607
-		err = f(&w, context);
1608
-		kunmap(v.bv_page);
1609
-		err;}), ({
1610
-		w = v;
1611
-		err = f(&w, context);})
1612
-	)
1613
-	return err;
1839
+	if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i)) &&
1840
+			 !iov_iter_is_kvec(i))
1841
+		return;
1842
+	i->iov_offset = state->iov_offset;
1843
+	i->count = state->count;
1844
+	/*
1845
+	 * For the *vec iters, nr_segs + iov is constant - if we increment
1846
+	 * the vec, then we also decrement the nr_segs count. Hence we don't
1847
+	 * need to track both of these, just one is enough and we can deduct
1848
+	 * the other from that. ITER_KVEC and ITER_IOVEC are the same struct
1849
+	 * size, so we can just increment the iov pointer as they are unionzed.
1850
+	 * ITER_BVEC _may_ be the same size on some archs, but on others it is
1851
+	 * not. Be safe and handle it separately.
1852
+	 */
1853
+	BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec));
1854
+	if (iov_iter_is_bvec(i))
1855
+		i->bvec -= state->nr_segs - i->nr_segs;
1856
+	else
1857
+		i->iov -= state->nr_segs - i->nr_segs;
1858
+	i->nr_segs = state->nr_segs;
16141859 }
1615
-EXPORT_SYMBOL(iov_iter_for_each_range);