| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * vhost transport for vsock |
|---|
| 3 | 4 | * |
|---|
| 4 | 5 | * Copyright (C) 2013-2015 Red Hat, Inc. |
|---|
| 5 | 6 | * Author: Asias He <asias@redhat.com> |
|---|
| 6 | 7 | * Stefan Hajnoczi <stefanha@redhat.com> |
|---|
| 7 | | - * |
|---|
| 8 | | - * This work is licensed under the terms of the GNU GPL, version 2. |
|---|
| 9 | 8 | */ |
|---|
| 10 | 9 | #include <linux/miscdevice.h> |
|---|
| 11 | 10 | #include <linux/atomic.h> |
|---|
| .. | .. |
|---|
| 35 | 34 | }; |
|---|
| 36 | 35 | |
|---|
| 37 | 36 | /* Used to track all the vhost_vsock instances on the system. */ |
|---|
| 38 | | -static DEFINE_SPINLOCK(vhost_vsock_lock); |
|---|
| 37 | +static DEFINE_MUTEX(vhost_vsock_mutex); |
|---|
| 39 | 38 | static DEFINE_READ_MOSTLY_HASHTABLE(vhost_vsock_hash, 8); |
|---|
| 40 | 39 | |
|---|
| 41 | 40 | struct vhost_vsock { |
|---|
| 42 | 41 | struct vhost_dev dev; |
|---|
| 43 | 42 | struct vhost_virtqueue vqs[2]; |
|---|
| 44 | 43 | |
|---|
| 45 | | - /* Link to global vhost_vsock_hash, writes use vhost_vsock_lock */ |
|---|
| 44 | + /* Link to global vhost_vsock_hash, writes use vhost_vsock_mutex */ |
|---|
| 46 | 45 | struct hlist_node hash; |
|---|
| 47 | 46 | |
|---|
| 48 | 47 | struct vhost_work send_pkt_work; |
|---|
| .. | .. |
|---|
| 59 | 58 | return VHOST_VSOCK_DEFAULT_HOST_CID; |
|---|
| 60 | 59 | } |
|---|
| 61 | 60 | |
|---|
| 62 | | -/* Callers that dereference the return value must hold vhost_vsock_lock or the |
|---|
| 61 | +/* Callers that dereference the return value must hold vhost_vsock_mutex or the |
|---|
| 63 | 62 | * RCU read lock. |
|---|
| 64 | 63 | */ |
|---|
| 65 | 64 | static struct vhost_vsock *vhost_vsock_get(u32 guest_cid) |
|---|
| .. | .. |
|---|
| 92 | 91 | |
|---|
| 93 | 92 | mutex_lock(&vq->mutex); |
|---|
| 94 | 93 | |
|---|
| 95 | | - if (!vq->private_data) |
|---|
| 94 | + if (!vhost_vq_get_backend(vq)) |
|---|
| 96 | 95 | goto out; |
|---|
| 97 | 96 | |
|---|
| 98 | 97 | /* Avoid further vmexits, we're already processing the virtqueue */ |
|---|
| .. | .. |
|---|
| 197 | 196 | * to send it with the next available buffer. |
|---|
| 198 | 197 | */ |
|---|
| 199 | 198 | if (pkt->off < pkt->len) { |
|---|
| 199 | + /* We are queueing the same virtio_vsock_pkt to handle |
|---|
| 200 | + * the remaining bytes, and we want to deliver it |
|---|
| 201 | + * to monitoring devices in the next iteration. |
|---|
| 202 | + */ |
|---|
| 203 | + pkt->tap_delivered = false; |
|---|
| 204 | + |
|---|
| 200 | 205 | spin_lock_bh(&vsock->send_pkt_list_lock); |
|---|
| 201 | 206 | list_add(&pkt->list, &vsock->send_pkt_list); |
|---|
| 202 | 207 | spin_unlock_bh(&vsock->send_pkt_list_lock); |
|---|
| .. | .. |
|---|
| 354 | 359 | return NULL; |
|---|
| 355 | 360 | } |
|---|
| 356 | 361 | |
|---|
| 357 | | - pkt->buf = kmalloc(pkt->len, GFP_KERNEL); |
|---|
| 362 | + pkt->buf = kvmalloc(pkt->len, GFP_KERNEL); |
|---|
| 358 | 363 | if (!pkt->buf) { |
|---|
| 359 | 364 | kfree(pkt); |
|---|
| 360 | 365 | return NULL; |
|---|
| 361 | 366 | } |
|---|
| 367 | + |
|---|
| 368 | + pkt->buf_len = pkt->len; |
|---|
| 362 | 369 | |
|---|
| 363 | 370 | nbytes = copy_from_iter(pkt->buf, pkt->len, &iov_iter); |
|---|
| 364 | 371 | if (nbytes != pkt->len) { |
|---|
| .. | .. |
|---|
| 385 | 392 | |
|---|
| 386 | 393 | static struct virtio_transport vhost_transport = { |
|---|
| 387 | 394 | .transport = { |
|---|
| 395 | + .module = THIS_MODULE, |
|---|
| 396 | + |
|---|
| 388 | 397 | .get_local_cid = vhost_transport_get_local_cid, |
|---|
| 389 | 398 | |
|---|
| 390 | 399 | .init = virtio_transport_do_socket_init, |
|---|
| .. | .. |
|---|
| 417 | 426 | .notify_send_pre_block = virtio_transport_notify_send_pre_block, |
|---|
| 418 | 427 | .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue, |
|---|
| 419 | 428 | .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue, |
|---|
| 429 | + .notify_buffer_size = virtio_transport_notify_buffer_size, |
|---|
| 420 | 430 | |
|---|
| 421 | | - .set_buffer_size = virtio_transport_set_buffer_size, |
|---|
| 422 | | - .set_min_buffer_size = virtio_transport_set_min_buffer_size, |
|---|
| 423 | | - .set_max_buffer_size = virtio_transport_set_max_buffer_size, |
|---|
| 424 | | - .get_buffer_size = virtio_transport_get_buffer_size, |
|---|
| 425 | | - .get_min_buffer_size = virtio_transport_get_min_buffer_size, |
|---|
| 426 | | - .get_max_buffer_size = virtio_transport_get_max_buffer_size, |
|---|
| 427 | 431 | }, |
|---|
| 428 | 432 | |
|---|
| 429 | 433 | .send_pkt = vhost_transport_send_pkt, |
|---|
| .. | .. |
|---|
| 442 | 446 | |
|---|
| 443 | 447 | mutex_lock(&vq->mutex); |
|---|
| 444 | 448 | |
|---|
| 445 | | - if (!vq->private_data) |
|---|
| 449 | + if (!vhost_vq_get_backend(vq)) |
|---|
| 446 | 450 | goto out; |
|---|
| 447 | 451 | |
|---|
| 448 | 452 | vhost_disable_notify(&vsock->dev, vq); |
|---|
| .. | .. |
|---|
| 535 | 539 | goto err_vq; |
|---|
| 536 | 540 | } |
|---|
| 537 | 541 | |
|---|
| 538 | | - if (!vq->private_data) { |
|---|
| 539 | | - vq->private_data = vsock; |
|---|
| 542 | + if (!vhost_vq_get_backend(vq)) { |
|---|
| 543 | + vhost_vq_set_backend(vq, vsock); |
|---|
| 540 | 544 | ret = vhost_vq_init_access(vq); |
|---|
| 541 | 545 | if (ret) |
|---|
| 542 | 546 | goto err_vq; |
|---|
| .. | .. |
|---|
| 554 | 558 | return 0; |
|---|
| 555 | 559 | |
|---|
| 556 | 560 | err_vq: |
|---|
| 557 | | - vq->private_data = NULL; |
|---|
| 561 | + vhost_vq_set_backend(vq, NULL); |
|---|
| 558 | 562 | mutex_unlock(&vq->mutex); |
|---|
| 559 | 563 | |
|---|
| 560 | 564 | for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { |
|---|
| 561 | 565 | vq = &vsock->vqs[i]; |
|---|
| 562 | 566 | |
|---|
| 563 | 567 | mutex_lock(&vq->mutex); |
|---|
| 564 | | - vq->private_data = NULL; |
|---|
| 568 | + vhost_vq_set_backend(vq, NULL); |
|---|
| 565 | 569 | mutex_unlock(&vq->mutex); |
|---|
| 566 | 570 | } |
|---|
| 567 | 571 | err: |
|---|
| .. | .. |
|---|
| 586 | 590 | struct vhost_virtqueue *vq = &vsock->vqs[i]; |
|---|
| 587 | 591 | |
|---|
| 588 | 592 | mutex_lock(&vq->mutex); |
|---|
| 589 | | - vq->private_data = NULL; |
|---|
| 593 | + vhost_vq_set_backend(vq, NULL); |
|---|
| 590 | 594 | mutex_unlock(&vq->mutex); |
|---|
| 591 | 595 | } |
|---|
| 592 | 596 | |
|---|
| .. | .. |
|---|
| 630 | 634 | |
|---|
| 631 | 635 | vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs), |
|---|
| 632 | 636 | UIO_MAXIOV, VHOST_VSOCK_PKT_WEIGHT, |
|---|
| 633 | | - VHOST_VSOCK_WEIGHT); |
|---|
| 637 | + VHOST_VSOCK_WEIGHT, true, NULL); |
|---|
| 634 | 638 | |
|---|
| 635 | 639 | file->private_data = vsock; |
|---|
| 636 | 640 | spin_lock_init(&vsock->send_pkt_list_lock); |
|---|
| .. | .. |
|---|
| 683 | 687 | { |
|---|
| 684 | 688 | struct vhost_vsock *vsock = file->private_data; |
|---|
| 685 | 689 | |
|---|
| 686 | | - spin_lock_bh(&vhost_vsock_lock); |
|---|
| 690 | + mutex_lock(&vhost_vsock_mutex); |
|---|
| 687 | 691 | if (vsock->guest_cid) |
|---|
| 688 | 692 | hash_del_rcu(&vsock->hash); |
|---|
| 689 | | - spin_unlock_bh(&vhost_vsock_lock); |
|---|
| 693 | + mutex_unlock(&vhost_vsock_mutex); |
|---|
| 690 | 694 | |
|---|
| 691 | 695 | /* Wait for other CPUs to finish using vsock */ |
|---|
| 692 | 696 | synchronize_rcu(); |
|---|
| .. | .. |
|---|
| 734 | 738 | if (guest_cid > U32_MAX) |
|---|
| 735 | 739 | return -EINVAL; |
|---|
| 736 | 740 | |
|---|
| 741 | + /* Refuse if CID is assigned to the guest->host transport (i.e. nested |
|---|
| 742 | + * VM), to make the loopback work. |
|---|
| 743 | + */ |
|---|
| 744 | + if (vsock_find_cid(guest_cid)) |
|---|
| 745 | + return -EADDRINUSE; |
|---|
| 746 | + |
|---|
| 737 | 747 | /* Refuse if CID is already in use */ |
|---|
| 738 | | - spin_lock_bh(&vhost_vsock_lock); |
|---|
| 748 | + mutex_lock(&vhost_vsock_mutex); |
|---|
| 739 | 749 | other = vhost_vsock_get(guest_cid); |
|---|
| 740 | 750 | if (other && other != vsock) { |
|---|
| 741 | | - spin_unlock_bh(&vhost_vsock_lock); |
|---|
| 751 | + mutex_unlock(&vhost_vsock_mutex); |
|---|
| 742 | 752 | return -EADDRINUSE; |
|---|
| 743 | 753 | } |
|---|
| 744 | 754 | |
|---|
| .. | .. |
|---|
| 747 | 757 | |
|---|
| 748 | 758 | vsock->guest_cid = guest_cid; |
|---|
| 749 | 759 | hash_add_rcu(vhost_vsock_hash, &vsock->hash, vsock->guest_cid); |
|---|
| 750 | | - spin_unlock_bh(&vhost_vsock_lock); |
|---|
| 760 | + mutex_unlock(&vhost_vsock_mutex); |
|---|
| 751 | 761 | |
|---|
| 752 | 762 | return 0; |
|---|
| 753 | 763 | } |
|---|
| .. | .. |
|---|
| 820 | 830 | } |
|---|
| 821 | 831 | } |
|---|
| 822 | 832 | |
|---|
| 823 | | -#ifdef CONFIG_COMPAT |
|---|
| 824 | | -static long vhost_vsock_dev_compat_ioctl(struct file *f, unsigned int ioctl, |
|---|
| 825 | | - unsigned long arg) |
|---|
| 826 | | -{ |
|---|
| 827 | | - return vhost_vsock_dev_ioctl(f, ioctl, (unsigned long)compat_ptr(arg)); |
|---|
| 828 | | -} |
|---|
| 829 | | -#endif |
|---|
| 830 | | - |
|---|
| 831 | 833 | static const struct file_operations vhost_vsock_fops = { |
|---|
| 832 | 834 | .owner = THIS_MODULE, |
|---|
| 833 | 835 | .open = vhost_vsock_dev_open, |
|---|
| 834 | 836 | .release = vhost_vsock_dev_release, |
|---|
| 835 | 837 | .llseek = noop_llseek, |
|---|
| 836 | 838 | .unlocked_ioctl = vhost_vsock_dev_ioctl, |
|---|
| 837 | | -#ifdef CONFIG_COMPAT |
|---|
| 838 | | - .compat_ioctl = vhost_vsock_dev_compat_ioctl, |
|---|
| 839 | | -#endif |
|---|
| 839 | + .compat_ioctl = compat_ptr_ioctl, |
|---|
| 840 | 840 | }; |
|---|
| 841 | 841 | |
|---|
| 842 | 842 | static struct miscdevice vhost_vsock_misc = { |
|---|
| .. | .. |
|---|
| 849 | 849 | { |
|---|
| 850 | 850 | int ret; |
|---|
| 851 | 851 | |
|---|
| 852 | | - ret = vsock_core_init(&vhost_transport.transport); |
|---|
| 852 | + ret = vsock_core_register(&vhost_transport.transport, |
|---|
| 853 | + VSOCK_TRANSPORT_F_H2G); |
|---|
| 853 | 854 | if (ret < 0) |
|---|
| 854 | 855 | return ret; |
|---|
| 855 | 856 | return misc_register(&vhost_vsock_misc); |
|---|
| .. | .. |
|---|
| 858 | 859 | static void __exit vhost_vsock_exit(void) |
|---|
| 859 | 860 | { |
|---|
| 860 | 861 | misc_deregister(&vhost_vsock_misc); |
|---|
| 861 | | - vsock_core_exit(); |
|---|
| 862 | + vsock_core_unregister(&vhost_transport.transport); |
|---|
| 862 | 863 | }; |
|---|
| 863 | 864 | |
|---|
| 864 | 865 | module_init(vhost_vsock_init); |
|---|