forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-05-10 cde9070d9970eef1f7ec2360586c802a16230ad8 
 kernel/drivers/vhost/vhost.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /* Copyright (C) 2009 Red Hat, Inc.
23  * Copyright (C) 2006 Rusty Russell IBM Corporation
34  *
....@@ -6,8 +7,6 @@
67  * Inspiration, some code, and most witty comments come from
78  * Documentation/virtual/lguest/lguest.c, by Rusty Russell
89  *
9
- * This work is licensed under the terms of the GNU GPL, version 2.
10
- *
1110  * Generic code for virtio server in host kernel.
1211  */
1312 
....@@ -15,7 +14,6 @@
1514 #include <linux/vhost.h>
1615 #include <linux/uio.h>
1716 #include <linux/mm.h>
18
-#include <linux/mmu_context.h>
1917 #include <linux/miscdevice.h>
2018 #include <linux/mutex.h>
2119 #include <linux/poll.h>
....@@ -50,10 +48,6 @@
5048 
5149 #define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num])
5250 #define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num])
53
-
54
-INTERVAL_TREE_DEFINE(struct vhost_umem_node,
55
-		     rb, __u64, __subtree_last,
56
-		     START, LAST, static inline, vhost_umem_interval_tree);
5751 
5852 #ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY
5953 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
....@@ -171,11 +165,16 @@
171165 			     void *key)
172166 {
173167 	struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait);
168
+	struct vhost_work *work = &poll->work;
174169 
175170 	if (!(key_to_poll(key) & poll->mask))
176171 		return 0;
177172 
178
-	vhost_poll_queue(poll);
173
+	if (!poll->dev->use_worker)
174
+		work->fn(work);
175
+	else
176
+		vhost_poll_queue(poll);
177
+
179178 	return 0;
180179 }
181180 
....@@ -205,7 +204,6 @@
205204 int vhost_poll_start(struct vhost_poll *poll, struct file *file)
206205 {
207206 	__poll_t mask;
208
-	int ret = 0;
209207 
210208 	if (poll->wqh)
211209 		return 0;
....@@ -215,10 +213,10 @@
215213 		vhost_poll_wakeup(&poll->wait, 0, 0, poll_to_key(mask));
216214 	if (mask & EPOLLERR) {
217215 		vhost_poll_stop(poll);
218
-		ret = -EINVAL;
216
+		return -EINVAL;
219217 	}
220218 
221
-	return ret;
219
+	return 0;
222220 }
223221 EXPORT_SYMBOL_GPL(vhost_poll_start);
224222 
....@@ -300,6 +298,18 @@
300298 		__vhost_vq_meta_reset(d->vqs[i]);
301299 }
302300 
301
+static void vhost_vring_call_reset(struct vhost_vring_call *call_ctx)
302
+{
303
+	call_ctx->ctx = NULL;
304
+	memset(&call_ctx->producer, 0x0, sizeof(struct irq_bypass_producer));
305
+}
306
+
307
+bool vhost_vq_is_setup(struct vhost_virtqueue *vq)
308
+{
309
+	return vq->avail && vq->desc && vq->used && vhost_vq_access_ok(vq);
310
+}
311
+EXPORT_SYMBOL_GPL(vhost_vq_is_setup);
312
+
303313 static void vhost_vq_reset(struct vhost_dev *dev,
304314 			   struct vhost_virtqueue *vq)
305315 {
....@@ -321,13 +331,13 @@
321331 	vq->log_base = NULL;
322332 	vq->error_ctx = NULL;
323333 	vq->kick = NULL;
324
-	vq->call_ctx = NULL;
325334 	vq->log_ctx = NULL;
326335 	vhost_disable_cross_endian(vq);
327336 	vhost_reset_is_le(vq);
328337 	vq->busyloop_timeout = 0;
329338 	vq->umem = NULL;
330339 	vq->iotlb = NULL;
340
+	vhost_vring_call_reset(&vq->call_ctx);
331341 	__vhost_vq_meta_reset(vq);
332342 }
333343 
....@@ -336,10 +346,8 @@
336346 	struct vhost_dev *dev = data;
337347 	struct vhost_work *work, *work_next;
338348 	struct llist_node *node;
339
-	mm_segment_t oldfs = get_fs();
340349 
341
-	set_fs(USER_DS);
342
-	use_mm(dev->mm);
350
+	kthread_use_mm(dev->mm);
343351 
344352 	for (;;) {
345353 		/* mb paired w/ kthread_stop */
....@@ -367,8 +375,7 @@
367375 				schedule();
368376 		}
369377 	}
370
-	unuse_mm(dev->mm);
371
-	set_fs(oldfs);
378
+	kthread_unuse_mm(dev->mm);
372379 	return 0;
373380 }
374381 
....@@ -431,9 +438,38 @@
431438 }
432439 EXPORT_SYMBOL_GPL(vhost_exceeds_weight);
433440 
441
+static size_t vhost_get_avail_size(struct vhost_virtqueue *vq,
442
+				   unsigned int num)
443
+{
444
+	size_t event __maybe_unused =
445
+	       vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
446
+
447
+	return sizeof(*vq->avail) +
448
+	       sizeof(*vq->avail->ring) * num + event;
449
+}
450
+
451
+static size_t vhost_get_used_size(struct vhost_virtqueue *vq,
452
+				  unsigned int num)
453
+{
454
+	size_t event __maybe_unused =
455
+	       vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
456
+
457
+	return sizeof(*vq->used) +
458
+	       sizeof(*vq->used->ring) * num + event;
459
+}
460
+
461
+static size_t vhost_get_desc_size(struct vhost_virtqueue *vq,
462
+				  unsigned int num)
463
+{
464
+	return sizeof(*vq->desc) * num;
465
+}
466
+
434467 void vhost_dev_init(struct vhost_dev *dev,
435468 		    struct vhost_virtqueue **vqs, int nvqs,
436
-		    int iov_limit, int weight, int byte_weight)
469
+		    int iov_limit, int weight, int byte_weight,
470
+		    bool use_worker,
471
+		    int (*msg_handler)(struct vhost_dev *dev,
472
+				       struct vhost_iotlb_msg *msg))
437473 {
438474 	struct vhost_virtqueue *vq;
439475 	int i;
....@@ -449,6 +485,8 @@
449485 	dev->iov_limit = iov_limit;
450486 	dev->weight = weight;
451487 	dev->byte_weight = byte_weight;
488
+	dev->use_worker = use_worker;
489
+	dev->msg_handler = msg_handler;
452490 	init_llist_head(&dev->work_list);
453491 	init_waitqueue_head(&dev->wait);
454492 	INIT_LIST_HEAD(&dev->read_list);
....@@ -511,6 +549,36 @@
511549 }
512550 EXPORT_SYMBOL_GPL(vhost_dev_has_owner);
513551 
552
+static void vhost_attach_mm(struct vhost_dev *dev)
553
+{
554
+	/* No owner, become one */
555
+	if (dev->use_worker) {
556
+		dev->mm = get_task_mm(current);
557
+	} else {
558
+		/* vDPA device does not use worker thead, so there's
559
+		 * no need to hold the address space for mm. This help
560
+		 * to avoid deadlock in the case of mmap() which may
561
+		 * held the refcnt of the file and depends on release
562
+		 * method to remove vma.
563
+		 */
564
+		dev->mm = current->mm;
565
+		mmgrab(dev->mm);
566
+	}
567
+}
568
+
569
+static void vhost_detach_mm(struct vhost_dev *dev)
570
+{
571
+	if (!dev->mm)
572
+		return;
573
+
574
+	if (dev->use_worker)
575
+		mmput(dev->mm);
576
+	else
577
+		mmdrop(dev->mm);
578
+
579
+	dev->mm = NULL;
580
+}
581
+
514582 /* Caller should have device mutex */
515583 long vhost_dev_set_owner(struct vhost_dev *dev)
516584 {
....@@ -523,21 +591,24 @@
523591 		goto err_mm;
524592 	}
525593 
526
-	/* No owner, become one */
527
-	dev->mm = get_task_mm(current);
594
+	vhost_attach_mm(dev);
595
+
528596 	dev->kcov_handle = kcov_common_handle();
529
-	worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid);
530
-	if (IS_ERR(worker)) {
531
-		err = PTR_ERR(worker);
532
-		goto err_worker;
597
+	if (dev->use_worker) {
598
+		worker = kthread_create(vhost_worker, dev,
599
+					"vhost-%d", current->pid);
600
+		if (IS_ERR(worker)) {
601
+			err = PTR_ERR(worker);
602
+			goto err_worker;
603
+		}
604
+
605
+		dev->worker = worker;
606
+		wake_up_process(worker); /* avoid contributing to loadavg */
607
+
608
+		err = vhost_attach_cgroups(dev);
609
+		if (err)
610
+			goto err_cgroup;
533611 	}
534
-
535
-	dev->worker = worker;
536
-	wake_up_process(worker);	/* avoid contributing to loadavg */
537
-
538
-	err = vhost_attach_cgroups(dev);
539
-	if (err)
540
-		goto err_cgroup;
541612 
542613 	err = vhost_dev_alloc_iovecs(dev);
543614 	if (err)
....@@ -545,33 +616,37 @@
545616 
546617 	return 0;
547618 err_cgroup:
548
-	kthread_stop(worker);
549
-	dev->worker = NULL;
619
+	if (dev->worker) {
620
+		kthread_stop(dev->worker);
621
+		dev->worker = NULL;
622
+	}
550623 err_worker:
551
-	if (dev->mm)
552
-		mmput(dev->mm);
553
-	dev->mm = NULL;
624
+	vhost_detach_mm(dev);
554625 	dev->kcov_handle = 0;
555626 err_mm:
556627 	return err;
557628 }
558629 EXPORT_SYMBOL_GPL(vhost_dev_set_owner);
559630 
560
-struct vhost_umem *vhost_dev_reset_owner_prepare(void)
631
+static struct vhost_iotlb *iotlb_alloc(void)
561632 {
562
-	return kvzalloc(sizeof(struct vhost_umem), GFP_KERNEL);
633
+	return vhost_iotlb_alloc(max_iotlb_entries,
634
+				 VHOST_IOTLB_FLAG_RETIRE);
635
+}
636
+
637
+struct vhost_iotlb *vhost_dev_reset_owner_prepare(void)
638
+{
639
+	return iotlb_alloc();
563640 }
564641 EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare);
565642 
566643 /* Caller should have device mutex */
567
-void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_umem *umem)
644
+void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *umem)
568645 {
569646 	int i;
570647 
571648 	vhost_dev_cleanup(dev);
572649 
573
-	/* Restore memory to default empty mapping. */
574
-	INIT_LIST_HEAD(&umem->umem_list);
575650 	dev->umem = umem;
576651 	/* We don't need VQ locks below since vhost_dev_cleanup makes sure
577652 	 * VQs aren't running.
....@@ -594,29 +669,7 @@
594669 }
595670 EXPORT_SYMBOL_GPL(vhost_dev_stop);
596671 
597
-static void vhost_umem_free(struct vhost_umem *umem,
598
-			    struct vhost_umem_node *node)
599
-{
600
-	vhost_umem_interval_tree_remove(node, &umem->umem_tree);
601
-	list_del(&node->link);
602
-	kfree(node);
603
-	umem->numem--;
604
-}
605
-
606
-static void vhost_umem_clean(struct vhost_umem *umem)
607
-{
608
-	struct vhost_umem_node *node, *tmp;
609
-
610
-	if (!umem)
611
-		return;
612
-
613
-	list_for_each_entry_safe(node, tmp, &umem->umem_list, link)
614
-		vhost_umem_free(umem, node);
615
-
616
-	kvfree(umem);
617
-}
618
-
619
-static void vhost_clear_msg(struct vhost_dev *dev)
672
+void vhost_clear_msg(struct vhost_dev *dev)
620673 {
621674 	struct vhost_msg_node *node, *n;
622675 
....@@ -634,6 +687,7 @@
634687 
635688 	spin_unlock(&dev->iotlb_lock);
636689 }
690
+EXPORT_SYMBOL_GPL(vhost_clear_msg);
637691 
638692 void vhost_dev_cleanup(struct vhost_dev *dev)
639693 {
....@@ -644,8 +698,8 @@
644698 			eventfd_ctx_put(dev->vqs[i]->error_ctx);
645699 		if (dev->vqs[i]->kick)
646700 			fput(dev->vqs[i]->kick);
647
-		if (dev->vqs[i]->call_ctx)
648
-			eventfd_ctx_put(dev->vqs[i]->call_ctx);
701
+		if (dev->vqs[i]->call_ctx.ctx)
702
+			eventfd_ctx_put(dev->vqs[i]->call_ctx.ctx);
649703 		vhost_vq_reset(dev, dev->vqs[i]);
650704 	}
651705 	vhost_dev_free_iovecs(dev);
....@@ -653,9 +707,9 @@
653707 		eventfd_ctx_put(dev->log_ctx);
654708 	dev->log_ctx = NULL;
655709 	/* No one will access memory at this point */
656
-	vhost_umem_clean(dev->umem);
710
+	vhost_iotlb_free(dev->umem);
657711 	dev->umem = NULL;
658
-	vhost_umem_clean(dev->iotlb);
712
+	vhost_iotlb_free(dev->iotlb);
659713 	dev->iotlb = NULL;
660714 	vhost_clear_msg(dev);
661715 	wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM);
....@@ -665,9 +719,7 @@
665719 		dev->worker = NULL;
666720 		dev->kcov_handle = 0;
667721 	}
668
-	if (dev->mm)
669
-		mmput(dev->mm);
670
-	dev->mm = NULL;
722
+	vhost_detach_mm(dev);
671723 }
672724 EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
673725 
....@@ -680,7 +732,7 @@
680732 	    a + (unsigned long)log_base > ULONG_MAX)
681733 		return false;
682734 
683
-	return access_ok(VERIFY_WRITE, log_base + a,
735
+	return access_ok(log_base + a,
684736 			 (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8);
685737 }
686738 
....@@ -697,27 +749,26 @@
697749 }
698750 
699751 /* Caller should have vq mutex and device mutex. */
700
-static bool vq_memory_access_ok(void __user *log_base, struct vhost_umem *umem,
752
+static bool vq_memory_access_ok(void __user *log_base, struct vhost_iotlb *umem,
701753 				int log_all)
702754 {
703
-	struct vhost_umem_node *node;
755
+	struct vhost_iotlb_map *map;
704756 
705757 	if (!umem)
706758 		return false;
707759 
708
-	list_for_each_entry(node, &umem->umem_list, link) {
709
-		unsigned long a = node->userspace_addr;
760
+	list_for_each_entry(map, &umem->list, link) {
761
+		unsigned long a = map->addr;
710762 
711
-		if (vhost_overflow(node->userspace_addr, node->size))
763
+		if (vhost_overflow(map->addr, map->size))
712764 			return false;
713765 
714766 
715
-		if (!access_ok(VERIFY_WRITE, (void __user *)a,
716
-				    node->size))
767
+		if (!access_ok((void __user *)a, map->size))
717768 			return false;
718769 		else if (log_all && !log_access_ok(log_base,
719
-						   node->start,
720
-						   node->size))
770
+						   map->start,
771
+						   map->size))
721772 			return false;
722773 	}
723774 	return true;
....@@ -727,17 +778,17 @@
727778 					       u64 addr, unsigned int size,
728779 					       int type)
729780 {
730
-	const struct vhost_umem_node *node = vq->meta_iotlb[type];
781
+	const struct vhost_iotlb_map *map = vq->meta_iotlb[type];
731782 
732
-	if (!node)
783
+	if (!map)
733784 		return NULL;
734785 
735
-	return (void *)(uintptr_t)(node->userspace_addr + addr - node->start);
786
+	return (void __user *)(uintptr_t)(map->addr + addr - map->start);
736787 }
737788 
738789 /* Can we switch to this memory table? */
739790 /* Caller should have device mutex but not vq mutex */
740
-static bool memory_access_ok(struct vhost_dev *d, struct vhost_umem *umem,
791
+static bool memory_access_ok(struct vhost_dev *d, struct vhost_iotlb *umem,
741792 			     int log_all)
742793 {
743794 	int i;
....@@ -871,7 +922,7 @@
871922  * not happen in this case.
872923  */
873924 static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq,
874
-					    void *addr, unsigned int size,
925
+					    void __user *addr, unsigned int size,
875926 					    int type)
876927 {
877928 	void __user *uaddr = vhost_vq_meta_fetch(vq,
....@@ -884,7 +935,7 @@
884935 
885936 #define vhost_put_user(vq, x, ptr)		\
886937 ({ \
887
-	int ret = -EFAULT; \
938
+	int ret; \
888939 	if (!vq->iotlb) { \
889940 		ret = __put_user(x, ptr); \
890941 	} else { \
....@@ -898,6 +949,34 @@
898949 	} \
899950 	ret; \
900951 })
952
+
953
+static inline int vhost_put_avail_event(struct vhost_virtqueue *vq)
954
+{
955
+	return vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx),
956
+			      vhost_avail_event(vq));
957
+}
958
+
959
+static inline int vhost_put_used(struct vhost_virtqueue *vq,
960
+				 struct vring_used_elem *head, int idx,
961
+				 int count)
962
+{
963
+	return vhost_copy_to_user(vq, vq->used->ring + idx, head,
964
+				  count * sizeof(*head));
965
+}
966
+
967
+static inline int vhost_put_used_flags(struct vhost_virtqueue *vq)
968
+
969
+{
970
+	return vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags),
971
+			      &vq->used->flags);
972
+}
973
+
974
+static inline int vhost_put_used_idx(struct vhost_virtqueue *vq)
975
+
976
+{
977
+	return vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx),
978
+			      &vq->used->idx);
979
+}
901980 
902981 #define vhost_get_user(vq, x, ptr, type)		\
903982 ({ \
....@@ -937,45 +1016,41 @@
9371016 		mutex_unlock(&d->vqs[i]->mutex);
9381017 }
9391018 
940
-static int vhost_new_umem_range(struct vhost_umem *umem,
941
-				u64 start, u64 size, u64 end,
942
-				u64 userspace_addr, int perm)
1019
+static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq,
1020
+				      __virtio16 *idx)
9431021 {
944
-	struct vhost_umem_node *tmp, *node;
945
-
946
-	if (!size)
947
-		return -EFAULT;
948
-
949
-	node = kmalloc(sizeof(*node), GFP_ATOMIC);
950
-	if (!node)
951
-		return -ENOMEM;
952
-
953
-	if (umem->numem == max_iotlb_entries) {
954
-		tmp = list_first_entry(&umem->umem_list, typeof(*tmp), link);
955
-		vhost_umem_free(umem, tmp);
956
-	}
957
-
958
-	node->start = start;
959
-	node->size = size;
960
-	node->last = end;
961
-	node->userspace_addr = userspace_addr;
962
-	node->perm = perm;
963
-	INIT_LIST_HEAD(&node->link);
964
-	list_add_tail(&node->link, &umem->umem_list);
965
-	vhost_umem_interval_tree_insert(node, &umem->umem_tree);
966
-	umem->numem++;
967
-
968
-	return 0;
1022
+	return vhost_get_avail(vq, *idx, &vq->avail->idx);
9691023 }
9701024 
971
-static void vhost_del_umem_range(struct vhost_umem *umem,
972
-				 u64 start, u64 end)
1025
+static inline int vhost_get_avail_head(struct vhost_virtqueue *vq,
1026
+				       __virtio16 *head, int idx)
9731027 {
974
-	struct vhost_umem_node *node;
1028
+	return vhost_get_avail(vq, *head,
1029
+			       &vq->avail->ring[idx & (vq->num - 1)]);
1030
+}
9751031 
976
-	while ((node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
977
-							   start, end)))
978
-		vhost_umem_free(umem, node);
1032
+static inline int vhost_get_avail_flags(struct vhost_virtqueue *vq,
1033
+					__virtio16 *flags)
1034
+{
1035
+	return vhost_get_avail(vq, *flags, &vq->avail->flags);
1036
+}
1037
+
1038
+static inline int vhost_get_used_event(struct vhost_virtqueue *vq,
1039
+				       __virtio16 *event)
1040
+{
1041
+	return vhost_get_avail(vq, *event, vhost_used_event(vq));
1042
+}
1043
+
1044
+static inline int vhost_get_used_idx(struct vhost_virtqueue *vq,
1045
+				     __virtio16 *idx)
1046
+{
1047
+	return vhost_get_used(vq, *idx, &vq->used->idx);
1048
+}
1049
+
1050
+static inline int vhost_get_desc(struct vhost_virtqueue *vq,
1051
+				 struct vring_desc *desc, int idx)
1052
+{
1053
+	return vhost_copy_from_user(vq, desc, vq->desc + idx, sizeof(*desc));
9791054 }
9801055 
9811056 static void vhost_iotlb_notify_vq(struct vhost_dev *d,
....@@ -1008,10 +1083,10 @@
10081083 		return false;
10091084 
10101085 	if ((access & VHOST_ACCESS_RO) &&
1011
-	    !access_ok(VERIFY_READ, (void __user *)a, size))
1086
+	    !access_ok((void __user *)a, size))
10121087 		return false;
10131088 	if ((access & VHOST_ACCESS_WO) &&
1014
-	    !access_ok(VERIFY_WRITE, (void __user *)a, size))
1089
+	    !access_ok((void __user *)a, size))
10151090 		return false;
10161091 	return true;
10171092 }
....@@ -1034,9 +1109,9 @@
10341109 			break;
10351110 		}
10361111 		vhost_vq_meta_reset(dev);
1037
-		if (vhost_new_umem_range(dev->iotlb, msg->iova, msg->size,
1038
-					 msg->iova + msg->size - 1,
1039
-					 msg->uaddr, msg->perm)) {
1112
+		if (vhost_iotlb_add_range(dev->iotlb, msg->iova,
1113
+					  msg->iova + msg->size - 1,
1114
+					  msg->uaddr, msg->perm)) {
10401115 			ret = -ENOMEM;
10411116 			break;
10421117 		}
....@@ -1048,8 +1123,8 @@
10481123 			break;
10491124 		}
10501125 		vhost_vq_meta_reset(dev);
1051
-		vhost_del_umem_range(dev->iotlb, msg->iova,
1052
-				     msg->iova + msg->size - 1);
1126
+		vhost_iotlb_del_range(dev->iotlb, msg->iova,
1127
+				      msg->iova + msg->size - 1);
10531128 		break;
10541129 	default:
10551130 		ret = -EINVAL;
....@@ -1095,7 +1170,12 @@
10951170 		ret = -EINVAL;
10961171 		goto done;
10971172 	}
1098
-	if (vhost_process_iotlb_msg(dev, &msg)) {
1173
+
1174
+	if (dev->msg_handler)
1175
+		ret = dev->msg_handler(dev, &msg);
1176
+	else
1177
+		ret = vhost_process_iotlb_msg(dev, &msg);
1178
+	if (ret) {
10991179 		ret = -EFAULT;
11001180 		goto done;
11011181 	}
....@@ -1217,59 +1297,58 @@
12171297 }
12181298 
12191299 static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num,
1220
-			 struct vring_desc __user *desc,
1221
-			 struct vring_avail __user *avail,
1222
-			 struct vring_used __user *used)
1300
+			 vring_desc_t __user *desc,
1301
+			 vring_avail_t __user *avail,
1302
+			 vring_used_t __user *used)
12231303 
12241304 {
1225
-	size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
1305
+	/* If an IOTLB device is present, the vring addresses are
1306
+	 * GIOVAs. Access validation occurs at prefetch time. */
1307
+	if (vq->iotlb)
1308
+		return true;
12261309 
1227
-	return access_ok(VERIFY_READ, desc, num * sizeof *desc) &&
1228
-	       access_ok(VERIFY_READ, avail,
1229
-			 sizeof *avail + num * sizeof *avail->ring + s) &&
1230
-	       access_ok(VERIFY_WRITE, used,
1231
-			sizeof *used + num * sizeof *used->ring + s);
1310
+	return access_ok(desc, vhost_get_desc_size(vq, num)) &&
1311
+	       access_ok(avail, vhost_get_avail_size(vq, num)) &&
1312
+	       access_ok(used, vhost_get_used_size(vq, num));
12321313 }
12331314 
12341315 static void vhost_vq_meta_update(struct vhost_virtqueue *vq,
1235
-				 const struct vhost_umem_node *node,
1316
+				 const struct vhost_iotlb_map *map,
12361317 				 int type)
12371318 {
12381319 	int access = (type == VHOST_ADDR_USED) ?
12391320 		     VHOST_ACCESS_WO : VHOST_ACCESS_RO;
12401321 
1241
-	if (likely(node->perm & access))
1242
-		vq->meta_iotlb[type] = node;
1322
+	if (likely(map->perm & access))
1323
+		vq->meta_iotlb[type] = map;
12431324 }
12441325 
12451326 static bool iotlb_access_ok(struct vhost_virtqueue *vq,
12461327 			    int access, u64 addr, u64 len, int type)
12471328 {
1248
-	const struct vhost_umem_node *node;
1249
-	struct vhost_umem *umem = vq->iotlb;
1329
+	const struct vhost_iotlb_map *map;
1330
+	struct vhost_iotlb *umem = vq->iotlb;
12501331 	u64 s = 0, size, orig_addr = addr, last = addr + len - 1;
12511332 
12521333 	if (vhost_vq_meta_fetch(vq, addr, len, type))
12531334 		return true;
12541335 
12551336 	while (len > s) {
1256
-		node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
1257
-							   addr,
1258
-							   last);
1259
-		if (node == NULL || node->start > addr) {
1337
+		map = vhost_iotlb_itree_first(umem, addr, last);
1338
+		if (map == NULL || map->start > addr) {
12601339 			vhost_iotlb_miss(vq, addr, access);
12611340 			return false;
1262
-		} else if (!(node->perm & access)) {
1341
+		} else if (!(map->perm & access)) {
12631342 			/* Report the possible access violation by
12641343 			 * request another translation from userspace.
12651344 			 */
12661345 			return false;
12671346 		}
12681347 
1269
-		size = node->size - addr + node->start;
1348
+		size = map->size - addr + map->start;
12701349 
12711350 		if (orig_addr == addr && size >= len)
1272
-			vhost_vq_meta_update(vq, node, type);
1351
+			vhost_vq_meta_update(vq, map, type);
12731352 
12741353 		s += size;
12751354 		addr += size;
....@@ -1278,26 +1357,22 @@
12781357 	return true;
12791358 }
12801359 
1281
-int vq_iotlb_prefetch(struct vhost_virtqueue *vq)
1360
+int vq_meta_prefetch(struct vhost_virtqueue *vq)
12821361 {
1283
-	size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
12841362 	unsigned int num = vq->num;
12851363 
12861364 	if (!vq->iotlb)
12871365 		return 1;
12881366 
1289
-	return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc,
1290
-			       num * sizeof(*vq->desc), VHOST_ADDR_DESC) &&
1291
-	       iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->avail,
1292
-			       sizeof *vq->avail +
1293
-			       num * sizeof(*vq->avail->ring) + s,
1367
+	return iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->desc,
1368
+			       vhost_get_desc_size(vq, num), VHOST_ADDR_DESC) &&
1369
+	       iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->avail,
1370
+			       vhost_get_avail_size(vq, num),
12941371 			       VHOST_ADDR_AVAIL) &&
1295
-	       iotlb_access_ok(vq, VHOST_ACCESS_WO, (u64)(uintptr_t)vq->used,
1296
-			       sizeof *vq->used +
1297
-			       num * sizeof(*vq->used->ring) + s,
1298
-			       VHOST_ADDR_USED);
1372
+	       iotlb_access_ok(vq, VHOST_MAP_WO, (u64)(uintptr_t)vq->used,
1373
+			       vhost_get_used_size(vq, num), VHOST_ADDR_USED);
12991374 }
1300
-EXPORT_SYMBOL_GPL(vq_iotlb_prefetch);
1375
+EXPORT_SYMBOL_GPL(vq_meta_prefetch);
13011376 
13021377 /* Can we log writes? */
13031378 /* Caller should have device mutex but not vq mutex */
....@@ -1307,18 +1382,28 @@
13071382 }
13081383 EXPORT_SYMBOL_GPL(vhost_log_access_ok);
13091384 
1385
+static bool vq_log_used_access_ok(struct vhost_virtqueue *vq,
1386
+				  void __user *log_base,
1387
+				  bool log_used,
1388
+				  u64 log_addr)
1389
+{
1390
+	/* If an IOTLB device is present, log_addr is a GIOVA that
1391
+	 * will never be logged by log_used(). */
1392
+	if (vq->iotlb)
1393
+		return true;
1394
+
1395
+	return !log_used || log_access_ok(log_base, log_addr,
1396
+					  vhost_get_used_size(vq, vq->num));
1397
+}
1398
+
13101399 /* Verify access for write logging. */
13111400 /* Caller should have vq mutex and device mutex */
13121401 static bool vq_log_access_ok(struct vhost_virtqueue *vq,
13131402 			     void __user *log_base)
13141403 {
1315
-	size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
1316
-
13171404 	return vq_memory_access_ok(log_base, vq->umem,
13181405 				   vhost_has_feature(vq, VHOST_F_LOG_ALL)) &&
1319
-		(!vq->log_used || log_access_ok(log_base, vq->log_addr,
1320
-					sizeof *vq->used +
1321
-					vq->num * sizeof *vq->used->ring + s));
1406
+		vq_log_used_access_ok(vq, log_base, vq->log_used, vq->log_addr);
13221407 }
13231408 
13241409 /* Can we start vq? */
....@@ -1328,33 +1413,15 @@
13281413 	if (!vq_log_access_ok(vq, vq->log_base))
13291414 		return false;
13301415 
1331
-	/* Access validation occurs at prefetch time with IOTLB */
1332
-	if (vq->iotlb)
1333
-		return true;
1334
-
13351416 	return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used);
13361417 }
13371418 EXPORT_SYMBOL_GPL(vhost_vq_access_ok);
1338
-
1339
-static struct vhost_umem *vhost_umem_alloc(void)
1340
-{
1341
-	struct vhost_umem *umem = kvzalloc(sizeof(*umem), GFP_KERNEL);
1342
-
1343
-	if (!umem)
1344
-		return NULL;
1345
-
1346
-	umem->umem_tree = RB_ROOT_CACHED;
1347
-	umem->numem = 0;
1348
-	INIT_LIST_HEAD(&umem->umem_list);
1349
-
1350
-	return umem;
1351
-}
13521419 
13531420 static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
13541421 {
13551422 	struct vhost_memory mem, *newmem;
13561423 	struct vhost_memory_region *region;
1357
-	struct vhost_umem *newumem, *oldumem;
1424
+	struct vhost_iotlb *newumem, *oldumem;
13581425 	unsigned long size = offsetof(struct vhost_memory, regions);
13591426 	int i;
13601427 
....@@ -1371,12 +1438,12 @@
13711438 
13721439 	memcpy(newmem, &mem, size);
13731440 	if (copy_from_user(newmem->regions, m->regions,
1374
-			   mem.nregions * sizeof *m->regions)) {
1441
+			   flex_array_size(newmem, regions, mem.nregions))) {
13751442 		kvfree(newmem);
13761443 		return -EFAULT;
13771444 	}
13781445 
1379
-	newumem = vhost_umem_alloc();
1446
+	newumem = iotlb_alloc();
13801447 	if (!newumem) {
13811448 		kvfree(newmem);
13821449 		return -ENOMEM;
....@@ -1385,13 +1452,12 @@
13851452 	for (region = newmem->regions;
13861453 	     region < newmem->regions + mem.nregions;
13871454 	     region++) {
1388
-		if (vhost_new_umem_range(newumem,
1389
-					 region->guest_phys_addr,
1390
-					 region->memory_size,
1391
-					 region->guest_phys_addr +
1392
-					 region->memory_size - 1,
1393
-					 region->userspace_addr,
1394
-					 VHOST_ACCESS_RW))
1455
+		if (vhost_iotlb_add_range(newumem,
1456
+					  region->guest_phys_addr,
1457
+					  region->guest_phys_addr +
1458
+					  region->memory_size - 1,
1459
+					  region->userspace_addr,
1460
+					  VHOST_MAP_RW))
13951461 			goto err;
13961462 	}
13971463 
....@@ -1409,15 +1475,112 @@
14091475 	}
14101476 
14111477 	kvfree(newmem);
1412
-	vhost_umem_clean(oldumem);
1478
+	vhost_iotlb_free(oldumem);
14131479 	return 0;
14141480 
14151481 err:
1416
-	vhost_umem_clean(newumem);
1482
+	vhost_iotlb_free(newumem);
14171483 	kvfree(newmem);
14181484 	return -EFAULT;
14191485 }
14201486 
1487
+static long vhost_vring_set_num(struct vhost_dev *d,
1488
+				struct vhost_virtqueue *vq,
1489
+				void __user *argp)
1490
+{
1491
+	struct vhost_vring_state s;
1492
+
1493
+	/* Resizing ring with an active backend?
1494
+	 * You don't want to do that. */
1495
+	if (vq->private_data)
1496
+		return -EBUSY;
1497
+
1498
+	if (copy_from_user(&s, argp, sizeof s))
1499
+		return -EFAULT;
1500
+
1501
+	if (!s.num || s.num > 0xffff || (s.num & (s.num - 1)))
1502
+		return -EINVAL;
1503
+	vq->num = s.num;
1504
+
1505
+	return 0;
1506
+}
1507
+
1508
+static long vhost_vring_set_addr(struct vhost_dev *d,
1509
+				 struct vhost_virtqueue *vq,
1510
+				 void __user *argp)
1511
+{
1512
+	struct vhost_vring_addr a;
1513
+
1514
+	if (copy_from_user(&a, argp, sizeof a))
1515
+		return -EFAULT;
1516
+	if (a.flags & ~(0x1 << VHOST_VRING_F_LOG))
1517
+		return -EOPNOTSUPP;
1518
+
1519
+	/* For 32bit, verify that the top 32bits of the user
1520
+	   data are set to zero. */
1521
+	if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr ||
1522
+	    (u64)(unsigned long)a.used_user_addr != a.used_user_addr ||
1523
+	    (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr)
1524
+		return -EFAULT;
1525
+
1526
+	/* Make sure it's safe to cast pointers to vring types. */
1527
+	BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE);
1528
+	BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE);
1529
+	if ((a.avail_user_addr & (VRING_AVAIL_ALIGN_SIZE - 1)) ||
1530
+	    (a.used_user_addr & (VRING_USED_ALIGN_SIZE - 1)) ||
1531
+	    (a.log_guest_addr & (VRING_USED_ALIGN_SIZE - 1)))
1532
+		return -EINVAL;
1533
+
1534
+	/* We only verify access here if backend is configured.
1535
+	 * If it is not, we don't as size might not have been setup.
1536
+	 * We will verify when backend is configured. */
1537
+	if (vq->private_data) {
1538
+		if (!vq_access_ok(vq, vq->num,
1539
+			(void __user *)(unsigned long)a.desc_user_addr,
1540
+			(void __user *)(unsigned long)a.avail_user_addr,
1541
+			(void __user *)(unsigned long)a.used_user_addr))
1542
+			return -EINVAL;
1543
+
1544
+		/* Also validate log access for used ring if enabled. */
1545
+		if (!vq_log_used_access_ok(vq, vq->log_base,
1546
+				a.flags & (0x1 << VHOST_VRING_F_LOG),
1547
+				a.log_guest_addr))
1548
+			return -EINVAL;
1549
+	}
1550
+
1551
+	vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG));
1552
+	vq->desc = (void __user *)(unsigned long)a.desc_user_addr;
1553
+	vq->avail = (void __user *)(unsigned long)a.avail_user_addr;
1554
+	vq->log_addr = a.log_guest_addr;
1555
+	vq->used = (void __user *)(unsigned long)a.used_user_addr;
1556
+
1557
+	return 0;
1558
+}
1559
+
1560
+static long vhost_vring_set_num_addr(struct vhost_dev *d,
1561
+				     struct vhost_virtqueue *vq,
1562
+				     unsigned int ioctl,
1563
+				     void __user *argp)
1564
+{
1565
+	long r;
1566
+
1567
+	mutex_lock(&vq->mutex);
1568
+
1569
+	switch (ioctl) {
1570
+	case VHOST_SET_VRING_NUM:
1571
+		r = vhost_vring_set_num(d, vq, argp);
1572
+		break;
1573
+	case VHOST_SET_VRING_ADDR:
1574
+		r = vhost_vring_set_addr(d, vq, argp);
1575
+		break;
1576
+	default:
1577
+		BUG();
1578
+	}
1579
+
1580
+	mutex_unlock(&vq->mutex);
1581
+
1582
+	return r;
1583
+}
14211584 long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
14221585 {
14231586 	struct file *eventfp, *filep = NULL;
....@@ -1427,7 +1590,6 @@
14271590 	struct vhost_virtqueue *vq;
14281591 	struct vhost_vring_state s;
14291592 	struct vhost_vring_file f;
1430
-	struct vhost_vring_addr a;
14311593 	u32 idx;
14321594 	long r;
14331595 
....@@ -1440,26 +1602,14 @@
14401602 	idx = array_index_nospec(idx, d->nvqs);
14411603 	vq = d->vqs[idx];
14421604 
1605
+	if (ioctl == VHOST_SET_VRING_NUM ||
1606
+	    ioctl == VHOST_SET_VRING_ADDR) {
1607
+		return vhost_vring_set_num_addr(d, vq, ioctl, argp);
1608
+	}
1609
+
14431610 	mutex_lock(&vq->mutex);
14441611 
14451612 	switch (ioctl) {
1446
-	case VHOST_SET_VRING_NUM:
1447
-		/* Resizing ring with an active backend?
1448
-		 * You don't want to do that. */
1449
-		if (vq->private_data) {
1450
-			r = -EBUSY;
1451
-			break;
1452
-		}
1453
-		if (copy_from_user(&s, argp, sizeof s)) {
1454
-			r = -EFAULT;
1455
-			break;
1456
-		}
1457
-		if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) {
1458
-			r = -EINVAL;
1459
-			break;
1460
-		}
1461
-		vq->num = s.num;
1462
-		break;
14631613 	case VHOST_SET_VRING_BASE:
14641614 		/* Moving base with an active backend?
14651615 		 * You don't want to do that. */
....@@ -1471,82 +1621,34 @@
14711621 			r = -EFAULT;
14721622 			break;
14731623 		}
1474
-		if (s.num > 0xffff) {
1475
-			r = -EINVAL;
1476
-			break;
1624
+		if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
1625
+			vq->last_avail_idx = s.num & 0xffff;
1626
+			vq->last_used_idx = (s.num >> 16) & 0xffff;
1627
+		} else {
1628
+			if (s.num > 0xffff) {
1629
+				r = -EINVAL;
1630
+				break;
1631
+			}
1632
+			vq->last_avail_idx = s.num;
14771633 		}
1478
-		vq->last_avail_idx = s.num;
14791634 		/* Forget the cached index value. */
14801635 		vq->avail_idx = vq->last_avail_idx;
14811636 		break;
14821637 	case VHOST_GET_VRING_BASE:
14831638 		s.index = idx;
1484
-		s.num = vq->last_avail_idx;
1639
+		if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED))
1640
+			s.num = (u32)vq->last_avail_idx | ((u32)vq->last_used_idx << 16);
1641
+		else
1642
+			s.num = vq->last_avail_idx;
14851643 		if (copy_to_user(argp, &s, sizeof s))
14861644 			r = -EFAULT;
1487
-		break;
1488
-	case VHOST_SET_VRING_ADDR:
1489
-		if (copy_from_user(&a, argp, sizeof a)) {
1490
-			r = -EFAULT;
1491
-			break;
1492
-		}
1493
-		if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) {
1494
-			r = -EOPNOTSUPP;
1495
-			break;
1496
-		}
1497
-		/* For 32bit, verify that the top 32bits of the user
1498
-		   data are set to zero. */
1499
-		if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr ||
1500
-		    (u64)(unsigned long)a.used_user_addr != a.used_user_addr ||
1501
-		    (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) {
1502
-			r = -EFAULT;
1503
-			break;
1504
-		}
1505
-
1506
-		/* Make sure it's safe to cast pointers to vring types. */
1507
-		BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE);
1508
-		BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE);
1509
-		if ((a.avail_user_addr & (VRING_AVAIL_ALIGN_SIZE - 1)) ||
1510
-		    (a.used_user_addr & (VRING_USED_ALIGN_SIZE - 1)) ||
1511
-		    (a.log_guest_addr & (VRING_USED_ALIGN_SIZE - 1))) {
1512
-			r = -EINVAL;
1513
-			break;
1514
-		}
1515
-
1516
-		/* We only verify access here if backend is configured.
1517
-		 * If it is not, we don't as size might not have been setup.
1518
-		 * We will verify when backend is configured. */
1519
-		if (vq->private_data) {
1520
-			if (!vq_access_ok(vq, vq->num,
1521
-				(void __user *)(unsigned long)a.desc_user_addr,
1522
-				(void __user *)(unsigned long)a.avail_user_addr,
1523
-				(void __user *)(unsigned long)a.used_user_addr)) {
1524
-				r = -EINVAL;
1525
-				break;
1526
-			}
1527
-
1528
-			/* Also validate log access for used ring if enabled. */
1529
-			if ((a.flags & (0x1 << VHOST_VRING_F_LOG)) &&
1530
-			    !log_access_ok(vq->log_base, a.log_guest_addr,
1531
-					   sizeof *vq->used +
1532
-					   vq->num * sizeof *vq->used->ring)) {
1533
-				r = -EINVAL;
1534
-				break;
1535
-			}
1536
-		}
1537
-
1538
-		vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG));
1539
-		vq->desc = (void __user *)(unsigned long)a.desc_user_addr;
1540
-		vq->avail = (void __user *)(unsigned long)a.avail_user_addr;
1541
-		vq->log_addr = a.log_guest_addr;
1542
-		vq->used = (void __user *)(unsigned long)a.used_user_addr;
15431645 		break;
15441646 	case VHOST_SET_VRING_KICK:
15451647 		if (copy_from_user(&f, argp, sizeof f)) {
15461648 			r = -EFAULT;
15471649 			break;
15481650 		}
1549
-		eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
1651
+		eventfp = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_fget(f.fd);
15501652 		if (IS_ERR(eventfp)) {
15511653 			r = PTR_ERR(eventfp);
15521654 			break;
....@@ -1562,19 +1664,20 @@
15621664 			r = -EFAULT;
15631665 			break;
15641666 		}
1565
-		ctx = f.fd == -1 ? NULL : eventfd_ctx_fdget(f.fd);
1667
+		ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd);
15661668 		if (IS_ERR(ctx)) {
15671669 			r = PTR_ERR(ctx);
15681670 			break;
15691671 		}
1570
-		swap(ctx, vq->call_ctx);
1672
+
1673
+		swap(ctx, vq->call_ctx.ctx);
15711674 		break;
15721675 	case VHOST_SET_VRING_ERR:
15731676 		if (copy_from_user(&f, argp, sizeof f)) {
15741677 			r = -EFAULT;
15751678 			break;
15761679 		}
1577
-		ctx = f.fd == -1 ? NULL : eventfd_ctx_fdget(f.fd);
1680
+		ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd);
15781681 		if (IS_ERR(ctx)) {
15791682 			r = PTR_ERR(ctx);
15801683 			break;
....@@ -1625,10 +1728,10 @@
16251728 
16261729 int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled)
16271730 {
1628
-	struct vhost_umem *niotlb, *oiotlb;
1731
+	struct vhost_iotlb *niotlb, *oiotlb;
16291732 	int i;
16301733 
1631
-	niotlb = vhost_umem_alloc();
1734
+	niotlb = iotlb_alloc();
16321735 	if (!niotlb)
16331736 		return -ENOMEM;
16341737 
....@@ -1644,7 +1747,7 @@
16441747 		mutex_unlock(&vq->mutex);
16451748 	}
16461749 
1647
-	vhost_umem_clean(oiotlb);
1750
+	vhost_iotlb_free(oiotlb);
16481751 
16491752 	return 0;
16501753 }
....@@ -1699,7 +1802,7 @@
16991802 		r = get_user(fd, (int __user *)argp);
17001803 		if (r < 0)
17011804 			break;
1702
-		ctx = fd == -1 ? NULL : eventfd_ctx_fdget(fd);
1805
+		ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
17031806 		if (IS_ERR(ctx)) {
17041807 			r = PTR_ERR(ctx);
17051808 			break;
....@@ -1724,7 +1827,7 @@
17241827 
17251828 /* TODO: This is really inefficient.  We need something like get_user()
17261829  * (instruction directly accesses the data, with an exception table entry
1727
- * returning -EFAULT). See Documentation/x86/exception-tables.txt.
1830
+ * returning -EFAULT). See Documentation/x86/exception-tables.rst.
17281831  */
17291832 static int set_bit_to_user(int nr, void __user *addr)
17301833 {
....@@ -1734,15 +1837,14 @@
17341837 	int bit = nr + (log % PAGE_SIZE) * 8;
17351838 	int r;
17361839 
1737
-	r = get_user_pages_fast(log, 1, 1, &page);
1840
+	r = pin_user_pages_fast(log, 1, FOLL_WRITE, &page);
17381841 	if (r < 0)
17391842 		return r;
17401843 	BUG_ON(r != 1);
17411844 	base = kmap_atomic(page);
17421845 	set_bit(bit, base);
17431846 	kunmap_atomic(base);
1744
-	set_page_dirty_lock(page);
1745
-	put_page(page);
1847
+	unpin_user_pages_dirty_lock(&page, 1, true);
17461848 	return 0;
17471849 }
17481850 
....@@ -1774,8 +1876,8 @@
17741876 
17751877 static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len)
17761878 {
1777
-	struct vhost_umem *umem = vq->umem;
1778
-	struct vhost_umem_node *u;
1879
+	struct vhost_iotlb *umem = vq->umem;
1880
+	struct vhost_iotlb_map *u;
17791881 	u64 start, end, l, min;
17801882 	int r;
17811883 	bool hit = false;
....@@ -1785,16 +1887,15 @@
17851887 		/* More than one GPAs can be mapped into a single HVA. So
17861888 		 * iterate all possible umems here to be safe.
17871889 		 */
1788
-		list_for_each_entry(u, &umem->umem_list, link) {
1789
-			if (u->userspace_addr > hva - 1 + len ||
1790
-			    u->userspace_addr - 1 + u->size < hva)
1890
+		list_for_each_entry(u, &umem->list, link) {
1891
+			if (u->addr > hva - 1 + len ||
1892
+			    u->addr - 1 + u->size < hva)
17911893 				continue;
1792
-			start = max(u->userspace_addr, hva);
1793
-			end = min(u->userspace_addr - 1 + u->size,
1794
-				  hva - 1 + len);
1894
+			start = max(u->addr, hva);
1895
+			end = min(u->addr - 1 + u->size, hva - 1 + len);
17951896 			l = end - start + 1;
17961897 			r = log_write(vq->log_base,
1797
-				      u->start + start - u->userspace_addr,
1898
+				      u->start + start - u->addr,
17981899 				      l);
17991900 			if (r < 0)
18001901 				return r;
....@@ -1814,7 +1915,7 @@
18141915 
18151916 static int log_used(struct vhost_virtqueue *vq, u64 used_offset, u64 len)
18161917 {
1817
-	struct iovec iov[64];
1918
+	struct iovec *iov = vq->log_iov;
18181919 	int i, ret;
18191920 
18201921 	if (!vq->iotlb)
....@@ -1874,8 +1975,7 @@
18741975 static int vhost_update_used_flags(struct vhost_virtqueue *vq)
18751976 {
18761977 	void __user *used;
1877
-	if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags),
1878
-			   &vq->used->flags) < 0)
1978
+	if (vhost_put_used_flags(vq))
18791979 		return -EFAULT;
18801980 	if (unlikely(vq->log_used)) {
18811981 		/* Make sure the flag is seen before log. */
....@@ -1892,8 +1992,7 @@
18921992 
18931993 static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event)
18941994 {
1895
-	if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx),
1896
-			   vhost_avail_event(vq)))
1995
+	if (vhost_put_avail_event(vq))
18971996 		return -EFAULT;
18981997 	if (unlikely(vq->log_used)) {
18991998 		void __user *used;
....@@ -1925,11 +2024,11 @@
19252024 		goto err;
19262025 	vq->signalled_used_valid = false;
19272026 	if (!vq->iotlb &&
1928
-	    !access_ok(VERIFY_READ, &vq->used->idx, sizeof vq->used->idx)) {
2027
+	    !access_ok(&vq->used->idx, sizeof vq->used->idx)) {
19292028 		r = -EFAULT;
19302029 		goto err;
19312030 	}
1932
-	r = vhost_get_used(vq, last_used_idx, &vq->used->idx);
2031
+	r = vhost_get_used_idx(vq, &last_used_idx);
19332032 	if (r) {
19342033 		vq_err(vq, "Can't access used idx at %p\n",
19352034 		       &vq->used->idx);
....@@ -1947,11 +2046,11 @@
19472046 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
19482047 			  struct iovec iov[], int iov_size, int access)
19492048 {
1950
-	const struct vhost_umem_node *node;
2049
+	const struct vhost_iotlb_map *map;
19512050 	struct vhost_dev *dev = vq->dev;
1952
-	struct vhost_umem *umem = dev->iotlb ? dev->iotlb : dev->umem;
2051
+	struct vhost_iotlb *umem = dev->iotlb ? dev->iotlb : dev->umem;
19532052 	struct iovec *_iov;
1954
-	u64 s = 0;
2053
+	u64 s = 0, last = addr + len - 1;
19552054 	int ret = 0;
19562055 
19572056 	while ((u64)len > s) {
....@@ -1961,25 +2060,24 @@
19612060 			break;
19622061 		}
19632062 
1964
-		node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
1965
-							addr, addr + len - 1);
1966
-		if (node == NULL || node->start > addr) {
2063
+		map = vhost_iotlb_itree_first(umem, addr, last);
2064
+		if (map == NULL || map->start > addr) {
19672065 			if (umem != dev->iotlb) {
19682066 				ret = -EFAULT;
19692067 				break;
19702068 			}
19712069 			ret = -EAGAIN;
19722070 			break;
1973
-		} else if (!(node->perm & access)) {
2071
+		} else if (!(map->perm & access)) {
19742072 			ret = -EPERM;
19752073 			break;
19762074 		}
19772075 
19782076 		_iov = iov + ret;
1979
-		size = node->size - addr + node->start;
2077
+		size = map->size - addr + map->start;
19802078 		_iov->iov_len = min((u64)len - s, size);
19812079 		_iov->iov_base = (void __user *)(unsigned long)
1982
-			(node->userspace_addr + addr - node->start);
2080
+				 (map->addr + addr - map->start);
19832081 		s += size;
19842082 		addr += size;
19852083 		++ret;
....@@ -2035,11 +2133,6 @@
20352133 		return ret;
20362134 	}
20372135 	iov_iter_init(&from, READ, vq->indirect, ret, len);
2038
-
2039
-	/* We will use the result as an address to read from, so most
2040
-	 * architectures only need a compiler barrier here. */
2041
-	read_barrier_depends();
2042
-
20432136 	count = len / sizeof desc;
20442137 	/* Buffers are chained via a 16 bit next field, so
20452138 	 * we can have at most 2^16 of these. */
....@@ -2128,7 +2221,7 @@
21282221 	last_avail_idx = vq->last_avail_idx;
21292222 
21302223 	if (vq->avail_idx == vq->last_avail_idx) {
2131
-		if (unlikely(vhost_get_avail(vq, avail_idx, &vq->avail->idx))) {
2224
+		if (unlikely(vhost_get_avail_idx(vq, &avail_idx))) {
21322225 			vq_err(vq, "Failed to access avail idx at %p\n",
21332226 				&vq->avail->idx);
21342227 			return -EFAULT;
....@@ -2155,8 +2248,7 @@
21552248 
21562249 	/* Grab the next descriptor number they're advertising, and increment
21572250 	 * the index we've seen. */
2158
-	if (unlikely(vhost_get_avail(vq, ring_head,
2159
-		     &vq->avail->ring[last_avail_idx & (vq->num - 1)]))) {
2251
+	if (unlikely(vhost_get_avail_head(vq, &ring_head, last_avail_idx))) {
21602252 		vq_err(vq, "Failed to read head: idx %d address %p\n",
21612253 		       last_avail_idx,
21622254 		       &vq->avail->ring[last_avail_idx % vq->num]);
....@@ -2191,8 +2283,7 @@
21912283 			       i, vq->num, head);
21922284 			return -EINVAL;
21932285 		}
2194
-		ret = vhost_copy_from_user(vq, &desc, vq->desc + i,
2195
-					   sizeof desc);
2286
+		ret = vhost_get_desc(vq, &desc, i);
21962287 		if (unlikely(ret)) {
21972288 			vq_err(vq, "Failed to get descriptor: idx %d addr %p\n",
21982289 			       i, vq->desc + i);
....@@ -2279,22 +2370,13 @@
22792370 			    struct vring_used_elem *heads,
22802371 			    unsigned count)
22812372 {
2282
-	struct vring_used_elem __user *used;
2373
+	vring_used_elem_t __user *used;
22832374 	u16 old, new;
22842375 	int start;
22852376 
22862377 	start = vq->last_used_idx & (vq->num - 1);
22872378 	used = vq->used->ring + start;
2288
-	if (count == 1) {
2289
-		if (vhost_put_user(vq, heads[0].id, &used->id)) {
2290
-			vq_err(vq, "Failed to write used id");
2291
-			return -EFAULT;
2292
-		}
2293
-		if (vhost_put_user(vq, heads[0].len, &used->len)) {
2294
-			vq_err(vq, "Failed to write used len");
2295
-			return -EFAULT;
2296
-		}
2297
-	} else if (vhost_copy_to_user(vq, used, heads, count * sizeof *used)) {
2379
+	if (vhost_put_used(vq, heads, start, count)) {
22982380 		vq_err(vq, "Failed to write used");
22992381 		return -EFAULT;
23002382 	}
....@@ -2336,8 +2418,7 @@
23362418 
23372419 	/* Make sure buffer is written before we update index. */
23382420 	smp_wmb();
2339
-	if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx),
2340
-			   &vq->used->idx)) {
2421
+	if (vhost_put_used_idx(vq)) {
23412422 		vq_err(vq, "Failed to increment used idx");
23422423 		return -EFAULT;
23432424 	}
....@@ -2370,7 +2451,7 @@
23702451 
23712452 	if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
23722453 		__virtio16 flags;
2373
-		if (vhost_get_avail(vq, flags, &vq->avail->flags)) {
2454
+		if (vhost_get_avail_flags(vq, &flags)) {
23742455 			vq_err(vq, "Failed to get flags");
23752456 			return true;
23762457 		}
....@@ -2384,7 +2465,7 @@
23842465 	if (unlikely(!v))
23852466 		return true;
23862467 
2387
-	if (vhost_get_avail(vq, event, vhost_used_event(vq))) {
2468
+	if (vhost_get_used_event(vq, &event)) {
23882469 		vq_err(vq, "Failed to get used event idx");
23892470 		return true;
23902471 	}
....@@ -2395,8 +2476,8 @@
23952476 void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
23962477 {
23972478 	/* Signal the Guest tell them we used something up. */
2398
-	if (vq->call_ctx && vhost_notify(dev, vq))
2399
-		eventfd_signal(vq->call_ctx, 1);
2479
+	if (vq->call_ctx.ctx && vhost_notify(dev, vq))
2480
+		eventfd_signal(vq->call_ctx.ctx, 1);
24002481 }
24012482 EXPORT_SYMBOL_GPL(vhost_signal);
24022483 
....@@ -2429,7 +2510,7 @@
24292510 	if (vq->avail_idx != vq->last_avail_idx)
24302511 		return false;
24312512 
2432
-	r = vhost_get_avail(vq, avail_idx, &vq->avail->idx);
2513
+	r = vhost_get_avail_idx(vq, &avail_idx);
24332514 	if (unlikely(r))
24342515 		return false;
24352516 	vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
....@@ -2465,7 +2546,7 @@
24652546 	/* They could have slipped one in as we were doing that: make
24662547 	 * sure it's written, then check again. */
24672548 	smp_mb();
2468
-	r = vhost_get_avail(vq, avail_idx, &vq->avail->idx);
2549
+	r = vhost_get_avail_idx(vq, &avail_idx);
24692550 	if (r) {
24702551 		vq_err(vq, "Failed to check avail idx at %p: %d\n",
24712552 		       &vq->avail->idx, r);
....@@ -2487,7 +2568,7 @@
24872568 	if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
24882569 		r = vhost_update_used_flags(vq);
24892570 		if (r)
2490
-			vq_err(vq, "Failed to enable notification at %p: %d\n",
2571
+			vq_err(vq, "Failed to disable notification at %p: %d\n",
24912572 			       &vq->used->flags, r);
24922573 	}
24932574 }
....@@ -2536,6 +2617,21 @@
25362617 }
25372618 EXPORT_SYMBOL_GPL(vhost_dequeue_msg);
25382619 
2620
+void vhost_set_backend_features(struct vhost_dev *dev, u64 features)
2621
+{
2622
+	struct vhost_virtqueue *vq;
2623
+	int i;
2624
+
2625
+	mutex_lock(&dev->mutex);
2626
+	for (i = 0; i < dev->nvqs; ++i) {
2627
+		vq = dev->vqs[i];
2628
+		mutex_lock(&vq->mutex);
2629
+		vq->acked_backend_features = features;
2630
+		mutex_unlock(&vq->mutex);
2631
+	}
2632
+	mutex_unlock(&dev->mutex);
2633
+}
2634
+EXPORT_SYMBOL_GPL(vhost_set_backend_features);
25392635 
25402636 static int __init vhost_init(void)
25412637 {