.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
1 | 2 | #include <linux/etherdevice.h> |
---|
2 | 3 | #include <linux/if_tap.h> |
---|
3 | 4 | #include <linux/if_vlan.h> |
---|
.. | .. |
---|
340 | 341 | features |= tap->tap_features; |
---|
341 | 342 | if (netif_needs_gso(skb, features)) { |
---|
342 | 343 | struct sk_buff *segs = __skb_gso_segment(skb, features, false); |
---|
| 344 | + struct sk_buff *next; |
---|
343 | 345 | |
---|
344 | 346 | if (IS_ERR(segs)) |
---|
345 | 347 | goto drop; |
---|
.. | .. |
---|
351 | 353 | } |
---|
352 | 354 | |
---|
353 | 355 | consume_skb(skb); |
---|
354 | | - while (segs) { |
---|
355 | | - struct sk_buff *nskb = segs->next; |
---|
356 | | - |
---|
357 | | - segs->next = NULL; |
---|
358 | | - if (ptr_ring_produce(&q->ring, segs)) { |
---|
359 | | - kfree_skb(segs); |
---|
360 | | - kfree_skb_list(nskb); |
---|
| 356 | + skb_list_walk_safe(segs, skb, next) { |
---|
| 357 | + skb_mark_not_on_list(skb); |
---|
| 358 | + if (ptr_ring_produce(&q->ring, skb)) { |
---|
| 359 | + kfree_skb(skb); |
---|
| 360 | + kfree_skb_list(next); |
---|
361 | 361 | break; |
---|
362 | 362 | } |
---|
363 | | - segs = nskb; |
---|
364 | 363 | } |
---|
365 | 364 | } else { |
---|
366 | 365 | /* If we receive a partial checksum and the tap side |
---|
.. | .. |
---|
519 | 518 | goto err; |
---|
520 | 519 | } |
---|
521 | 520 | |
---|
522 | | - RCU_INIT_POINTER(q->sock.wq, &q->wq); |
---|
523 | | - init_waitqueue_head(&q->wq.wait); |
---|
| 521 | + init_waitqueue_head(&q->sock.wq.wait); |
---|
524 | 522 | q->sock.type = SOCK_RAW; |
---|
525 | 523 | q->sock.state = SS_CONNECTED; |
---|
526 | 524 | q->sock.file = file; |
---|
.. | .. |
---|
578 | 576 | goto out; |
---|
579 | 577 | |
---|
580 | 578 | mask = 0; |
---|
581 | | - poll_wait(file, &q->wq.wait, wait); |
---|
| 579 | + poll_wait(file, &q->sock.wq.wait, wait); |
---|
582 | 580 | |
---|
583 | 581 | if (!ptr_ring_empty(&q->ring)) |
---|
584 | 582 | mask |= EPOLLIN | EPOLLRDNORM; |
---|
.. | .. |
---|
619 | 617 | #define TAP_RESERVE HH_DATA_OFF(ETH_HLEN) |
---|
620 | 618 | |
---|
621 | 619 | /* Get packet from user space buffer */ |
---|
622 | | -static ssize_t tap_get_user(struct tap_queue *q, struct msghdr *m, |
---|
| 620 | +static ssize_t tap_get_user(struct tap_queue *q, void *msg_control, |
---|
623 | 621 | struct iov_iter *from, int noblock) |
---|
624 | 622 | { |
---|
625 | 623 | int good_linear = SKB_MAX_HEAD(TAP_RESERVE); |
---|
.. | .. |
---|
663 | 661 | if (unlikely(len < ETH_HLEN)) |
---|
664 | 662 | goto err; |
---|
665 | 663 | |
---|
666 | | - if (m && m->msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY)) { |
---|
| 664 | + if (msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY)) { |
---|
667 | 665 | struct iov_iter i; |
---|
668 | 666 | |
---|
669 | 667 | copylen = vnet_hdr.hdr_len ? |
---|
.. | .. |
---|
712 | 710 | goto err_kfree; |
---|
713 | 711 | } |
---|
714 | 712 | |
---|
715 | | - skb_probe_transport_header(skb, ETH_HLEN); |
---|
| 713 | + skb_probe_transport_header(skb); |
---|
716 | 714 | |
---|
717 | 715 | /* Move network header to the right position for VLAN tagged packets */ |
---|
718 | 716 | if ((skb->protocol == htons(ETH_P_8021Q) || |
---|
.. | .. |
---|
724 | 722 | tap = rcu_dereference(q->tap); |
---|
725 | 723 | /* copy skb_ubuf_info for callback when skb has no error */ |
---|
726 | 724 | if (zerocopy) { |
---|
727 | | - skb_shinfo(skb)->destructor_arg = m->msg_control; |
---|
| 725 | + skb_shinfo(skb)->destructor_arg = msg_control; |
---|
728 | 726 | skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY; |
---|
729 | 727 | skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG; |
---|
730 | | - } else if (m && m->msg_control) { |
---|
731 | | - struct ubuf_info *uarg = m->msg_control; |
---|
| 728 | + } else if (msg_control) { |
---|
| 729 | + struct ubuf_info *uarg = msg_control; |
---|
732 | 730 | uarg->callback(uarg, false); |
---|
733 | 731 | } |
---|
734 | 732 | |
---|
.. | .. |
---|
830 | 828 | ssize_t ret = 0; |
---|
831 | 829 | |
---|
832 | 830 | if (!iov_iter_count(to)) { |
---|
833 | | - if (skb) |
---|
834 | | - kfree_skb(skb); |
---|
| 831 | + kfree_skb(skb); |
---|
835 | 832 | return 0; |
---|
836 | 833 | } |
---|
837 | 834 | |
---|
.. | .. |
---|
1096 | 1093 | return -ENOLINK; |
---|
1097 | 1094 | } |
---|
1098 | 1095 | ret = 0; |
---|
1099 | | - u = tap->dev->type; |
---|
| 1096 | + dev_get_mac_address(&sa, dev_net(tap->dev), tap->dev->name); |
---|
1100 | 1097 | if (copy_to_user(&ifr->ifr_name, tap->dev->name, IFNAMSIZ) || |
---|
1101 | | - copy_to_user(&ifr->ifr_hwaddr.sa_data, tap->dev->dev_addr, ETH_ALEN) || |
---|
1102 | | - put_user(u, &ifr->ifr_hwaddr.sa_family)) |
---|
| 1098 | + copy_to_user(&ifr->ifr_hwaddr, &sa, sizeof(sa))) |
---|
1103 | 1099 | ret = -EFAULT; |
---|
1104 | 1100 | tap_put_tap_dev(tap); |
---|
1105 | 1101 | rtnl_unlock(); |
---|
.. | .. |
---|
1114 | 1110 | rtnl_unlock(); |
---|
1115 | 1111 | return -ENOLINK; |
---|
1116 | 1112 | } |
---|
1117 | | - ret = dev_set_mac_address(tap->dev, &sa); |
---|
| 1113 | + ret = dev_set_mac_address_user(tap->dev, &sa, NULL); |
---|
1118 | 1114 | tap_put_tap_dev(tap); |
---|
1119 | 1115 | rtnl_unlock(); |
---|
1120 | 1116 | return ret; |
---|
.. | .. |
---|
1123 | 1119 | return -EINVAL; |
---|
1124 | 1120 | } |
---|
1125 | 1121 | } |
---|
1126 | | - |
---|
1127 | | -#ifdef CONFIG_COMPAT |
---|
1128 | | -static long tap_compat_ioctl(struct file *file, unsigned int cmd, |
---|
1129 | | - unsigned long arg) |
---|
1130 | | -{ |
---|
1131 | | - return tap_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); |
---|
1132 | | -} |
---|
1133 | | -#endif |
---|
1134 | 1122 | |
---|
1135 | 1123 | static const struct file_operations tap_fops = { |
---|
1136 | 1124 | .owner = THIS_MODULE, |
---|
.. | .. |
---|
1141 | 1129 | .poll = tap_poll, |
---|
1142 | 1130 | .llseek = no_llseek, |
---|
1143 | 1131 | .unlocked_ioctl = tap_ioctl, |
---|
1144 | | -#ifdef CONFIG_COMPAT |
---|
1145 | | - .compat_ioctl = tap_compat_ioctl, |
---|
1146 | | -#endif |
---|
| 1132 | + .compat_ioctl = compat_ptr_ioctl, |
---|
1147 | 1133 | }; |
---|
| 1134 | + |
---|
| 1135 | +static int tap_get_user_xdp(struct tap_queue *q, struct xdp_buff *xdp) |
---|
| 1136 | +{ |
---|
| 1137 | + struct tun_xdp_hdr *hdr = xdp->data_hard_start; |
---|
| 1138 | + struct virtio_net_hdr *gso = &hdr->gso; |
---|
| 1139 | + int buflen = hdr->buflen; |
---|
| 1140 | + int vnet_hdr_len = 0; |
---|
| 1141 | + struct tap_dev *tap; |
---|
| 1142 | + struct sk_buff *skb; |
---|
| 1143 | + int err, depth; |
---|
| 1144 | + |
---|
| 1145 | + if (q->flags & IFF_VNET_HDR) |
---|
| 1146 | + vnet_hdr_len = READ_ONCE(q->vnet_hdr_sz); |
---|
| 1147 | + |
---|
| 1148 | + skb = build_skb(xdp->data_hard_start, buflen); |
---|
| 1149 | + if (!skb) { |
---|
| 1150 | + err = -ENOMEM; |
---|
| 1151 | + goto err; |
---|
| 1152 | + } |
---|
| 1153 | + |
---|
| 1154 | + skb_reserve(skb, xdp->data - xdp->data_hard_start); |
---|
| 1155 | + skb_put(skb, xdp->data_end - xdp->data); |
---|
| 1156 | + |
---|
| 1157 | + skb_set_network_header(skb, ETH_HLEN); |
---|
| 1158 | + skb_reset_mac_header(skb); |
---|
| 1159 | + skb->protocol = eth_hdr(skb)->h_proto; |
---|
| 1160 | + |
---|
| 1161 | + if (vnet_hdr_len) { |
---|
| 1162 | + err = virtio_net_hdr_to_skb(skb, gso, tap_is_little_endian(q)); |
---|
| 1163 | + if (err) |
---|
| 1164 | + goto err_kfree; |
---|
| 1165 | + } |
---|
| 1166 | + |
---|
| 1167 | + /* Move network header to the right position for VLAN tagged packets */ |
---|
| 1168 | + if ((skb->protocol == htons(ETH_P_8021Q) || |
---|
| 1169 | + skb->protocol == htons(ETH_P_8021AD)) && |
---|
| 1170 | + __vlan_get_protocol(skb, skb->protocol, &depth) != 0) |
---|
| 1171 | + skb_set_network_header(skb, depth); |
---|
| 1172 | + |
---|
| 1173 | + rcu_read_lock(); |
---|
| 1174 | + tap = rcu_dereference(q->tap); |
---|
| 1175 | + if (tap) { |
---|
| 1176 | + skb->dev = tap->dev; |
---|
| 1177 | + skb_probe_transport_header(skb); |
---|
| 1178 | + dev_queue_xmit(skb); |
---|
| 1179 | + } else { |
---|
| 1180 | + kfree_skb(skb); |
---|
| 1181 | + } |
---|
| 1182 | + rcu_read_unlock(); |
---|
| 1183 | + |
---|
| 1184 | + return 0; |
---|
| 1185 | + |
---|
| 1186 | +err_kfree: |
---|
| 1187 | + kfree_skb(skb); |
---|
| 1188 | +err: |
---|
| 1189 | + rcu_read_lock(); |
---|
| 1190 | + tap = rcu_dereference(q->tap); |
---|
| 1191 | + if (tap && tap->count_tx_dropped) |
---|
| 1192 | + tap->count_tx_dropped(tap); |
---|
| 1193 | + rcu_read_unlock(); |
---|
| 1194 | + return err; |
---|
| 1195 | +} |
---|
1148 | 1196 | |
---|
1149 | 1197 | static int tap_sendmsg(struct socket *sock, struct msghdr *m, |
---|
1150 | 1198 | size_t total_len) |
---|
1151 | 1199 | { |
---|
1152 | 1200 | struct tap_queue *q = container_of(sock, struct tap_queue, sock); |
---|
1153 | | - return tap_get_user(q, m, &m->msg_iter, m->msg_flags & MSG_DONTWAIT); |
---|
| 1201 | + struct tun_msg_ctl *ctl = m->msg_control; |
---|
| 1202 | + struct xdp_buff *xdp; |
---|
| 1203 | + int i; |
---|
| 1204 | + |
---|
| 1205 | + if (m->msg_controllen == sizeof(struct tun_msg_ctl) && |
---|
| 1206 | + ctl && ctl->type == TUN_MSG_PTR) { |
---|
| 1207 | + for (i = 0; i < ctl->num; i++) { |
---|
| 1208 | + xdp = &((struct xdp_buff *)ctl->ptr)[i]; |
---|
| 1209 | + tap_get_user_xdp(q, xdp); |
---|
| 1210 | + } |
---|
| 1211 | + return 0; |
---|
| 1212 | + } |
---|
| 1213 | + |
---|
| 1214 | + return tap_get_user(q, ctl ? ctl->ptr : NULL, &m->msg_iter, |
---|
| 1215 | + m->msg_flags & MSG_DONTWAIT); |
---|
1154 | 1216 | } |
---|
1155 | 1217 | |
---|
1156 | 1218 | static int tap_recvmsg(struct socket *sock, struct msghdr *m, |
---|
.. | .. |
---|
1160 | 1222 | struct sk_buff *skb = m->msg_control; |
---|
1161 | 1223 | int ret; |
---|
1162 | 1224 | if (flags & ~(MSG_DONTWAIT|MSG_TRUNC)) { |
---|
1163 | | - if (skb) |
---|
1164 | | - kfree_skb(skb); |
---|
| 1225 | + kfree_skb(skb); |
---|
1165 | 1226 | return -EINVAL; |
---|
1166 | 1227 | } |
---|
1167 | 1228 | ret = tap_do_read(q, &m->msg_iter, flags & MSG_DONTWAIT, skb); |
---|