.. | .. |
---|
269 | 269 | |
---|
270 | 270 | static bool packet_use_direct_xmit(const struct packet_sock *po) |
---|
271 | 271 | { |
---|
272 | | - return po->xmit == packet_direct_xmit; |
---|
| 272 | + /* Paired with WRITE_ONCE() in packet_setsockopt() */ |
---|
| 273 | + return READ_ONCE(po->xmit) == packet_direct_xmit; |
---|
273 | 274 | } |
---|
274 | 275 | |
---|
275 | 276 | static u16 packet_pick_tx_queue(struct sk_buff *skb) |
---|
.. | .. |
---|
365 | 366 | { |
---|
366 | 367 | union tpacket_uhdr h; |
---|
367 | 368 | |
---|
| 369 | + /* WRITE_ONCE() are paired with READ_ONCE() in __packet_get_status */ |
---|
| 370 | + |
---|
368 | 371 | h.raw = frame; |
---|
369 | 372 | switch (po->tp_version) { |
---|
370 | 373 | case TPACKET_V1: |
---|
371 | | - h.h1->tp_status = status; |
---|
| 374 | + WRITE_ONCE(h.h1->tp_status, status); |
---|
372 | 375 | flush_dcache_page(pgv_to_page(&h.h1->tp_status)); |
---|
373 | 376 | break; |
---|
374 | 377 | case TPACKET_V2: |
---|
375 | | - h.h2->tp_status = status; |
---|
| 378 | + WRITE_ONCE(h.h2->tp_status, status); |
---|
376 | 379 | flush_dcache_page(pgv_to_page(&h.h2->tp_status)); |
---|
377 | 380 | break; |
---|
378 | 381 | case TPACKET_V3: |
---|
379 | | - h.h3->tp_status = status; |
---|
| 382 | + WRITE_ONCE(h.h3->tp_status, status); |
---|
380 | 383 | flush_dcache_page(pgv_to_page(&h.h3->tp_status)); |
---|
381 | 384 | break; |
---|
382 | 385 | default: |
---|
.. | .. |
---|
393 | 396 | |
---|
394 | 397 | smp_rmb(); |
---|
395 | 398 | |
---|
| 399 | + /* READ_ONCE() are paired with WRITE_ONCE() in __packet_set_status */ |
---|
| 400 | + |
---|
396 | 401 | h.raw = frame; |
---|
397 | 402 | switch (po->tp_version) { |
---|
398 | 403 | case TPACKET_V1: |
---|
399 | 404 | flush_dcache_page(pgv_to_page(&h.h1->tp_status)); |
---|
400 | | - return h.h1->tp_status; |
---|
| 405 | + return READ_ONCE(h.h1->tp_status); |
---|
401 | 406 | case TPACKET_V2: |
---|
402 | 407 | flush_dcache_page(pgv_to_page(&h.h2->tp_status)); |
---|
403 | | - return h.h2->tp_status; |
---|
| 408 | + return READ_ONCE(h.h2->tp_status); |
---|
404 | 409 | case TPACKET_V3: |
---|
405 | 410 | flush_dcache_page(pgv_to_page(&h.h3->tp_status)); |
---|
406 | | - return h.h3->tp_status; |
---|
| 411 | + return READ_ONCE(h.h3->tp_status); |
---|
407 | 412 | default: |
---|
408 | 413 | WARN(1, "TPACKET version not supported.\n"); |
---|
409 | 414 | BUG(); |
---|
.. | .. |
---|
1885 | 1890 | |
---|
1886 | 1891 | static void packet_parse_headers(struct sk_buff *skb, struct socket *sock) |
---|
1887 | 1892 | { |
---|
| 1893 | + int depth; |
---|
| 1894 | + |
---|
1888 | 1895 | if ((!skb->protocol || skb->protocol == htons(ETH_P_ALL)) && |
---|
1889 | 1896 | sock->type == SOCK_RAW) { |
---|
1890 | 1897 | skb_reset_mac_header(skb); |
---|
1891 | 1898 | skb->protocol = dev_parse_header_protocol(skb); |
---|
1892 | 1899 | } |
---|
| 1900 | + |
---|
| 1901 | + /* Move network header to the right position for VLAN tagged packets */ |
---|
| 1902 | + if (likely(skb->dev->type == ARPHRD_ETHER) && |
---|
| 1903 | + eth_type_vlan(skb->protocol) && |
---|
| 1904 | + vlan_get_protocol_and_depth(skb, skb->protocol, &depth) != 0) |
---|
| 1905 | + skb_set_network_header(skb, depth); |
---|
1893 | 1906 | |
---|
1894 | 1907 | skb_probe_transport_header(skb); |
---|
1895 | 1908 | } |
---|
.. | .. |
---|
1985 | 1998 | goto retry; |
---|
1986 | 1999 | } |
---|
1987 | 2000 | |
---|
1988 | | - if (!dev_validate_header(dev, skb->data, len)) { |
---|
| 2001 | + if (!dev_validate_header(dev, skb->data, len) || !skb->len) { |
---|
1989 | 2002 | err = -EINVAL; |
---|
1990 | 2003 | goto out_unlock; |
---|
1991 | 2004 | } |
---|
.. | .. |
---|
2135 | 2148 | sll = &PACKET_SKB_CB(skb)->sa.ll; |
---|
2136 | 2149 | sll->sll_hatype = dev->type; |
---|
2137 | 2150 | sll->sll_pkttype = skb->pkt_type; |
---|
2138 | | - if (unlikely(po->origdev)) |
---|
| 2151 | + if (unlikely(packet_sock_flag(po, PACKET_SOCK_ORIGDEV))) |
---|
2139 | 2152 | sll->sll_ifindex = orig_dev->ifindex; |
---|
2140 | 2153 | else |
---|
2141 | 2154 | sll->sll_ifindex = dev->ifindex; |
---|
.. | .. |
---|
2408 | 2421 | sll->sll_hatype = dev->type; |
---|
2409 | 2422 | sll->sll_protocol = skb->protocol; |
---|
2410 | 2423 | sll->sll_pkttype = skb->pkt_type; |
---|
2411 | | - if (unlikely(po->origdev)) |
---|
| 2424 | + if (unlikely(packet_sock_flag(po, PACKET_SOCK_ORIGDEV))) |
---|
2412 | 2425 | sll->sll_ifindex = orig_dev->ifindex; |
---|
2413 | 2426 | else |
---|
2414 | 2427 | sll->sll_ifindex = dev->ifindex; |
---|
.. | .. |
---|
2815 | 2828 | packet_inc_pending(&po->tx_ring); |
---|
2816 | 2829 | |
---|
2817 | 2830 | status = TP_STATUS_SEND_REQUEST; |
---|
2818 | | - err = po->xmit(skb); |
---|
| 2831 | + /* Paired with WRITE_ONCE() in packet_setsockopt() */ |
---|
| 2832 | + err = READ_ONCE(po->xmit)(skb); |
---|
2819 | 2833 | if (unlikely(err != 0)) { |
---|
2820 | 2834 | if (err > 0) |
---|
2821 | 2835 | err = net_xmit_errno(err); |
---|
.. | .. |
---|
3005 | 3019 | skb->mark = sockc.mark; |
---|
3006 | 3020 | skb->tstamp = sockc.transmit_time; |
---|
3007 | 3021 | |
---|
| 3022 | + if (unlikely(extra_len == 4)) |
---|
| 3023 | + skb->no_fcs = 1; |
---|
| 3024 | + |
---|
| 3025 | + packet_parse_headers(skb, sock); |
---|
| 3026 | + |
---|
3008 | 3027 | if (has_vnet_hdr) { |
---|
3009 | 3028 | err = virtio_net_hdr_to_skb(skb, &vnet_hdr, vio_le()); |
---|
3010 | 3029 | if (err) |
---|
.. | .. |
---|
3013 | 3032 | virtio_net_hdr_set_proto(skb, &vnet_hdr); |
---|
3014 | 3033 | } |
---|
3015 | 3034 | |
---|
3016 | | - packet_parse_headers(skb, sock); |
---|
3017 | | - |
---|
3018 | | - if (unlikely(extra_len == 4)) |
---|
3019 | | - skb->no_fcs = 1; |
---|
3020 | | - |
---|
3021 | | - err = po->xmit(skb); |
---|
| 3035 | + /* Paired with WRITE_ONCE() in packet_setsockopt() */ |
---|
| 3036 | + err = READ_ONCE(po->xmit)(skb); |
---|
3022 | 3037 | if (unlikely(err != 0)) { |
---|
3023 | 3038 | if (err > 0) |
---|
3024 | 3039 | err = net_xmit_errno(err); |
---|
.. | .. |
---|
3146 | 3161 | |
---|
3147 | 3162 | lock_sock(sk); |
---|
3148 | 3163 | spin_lock(&po->bind_lock); |
---|
| 3164 | + if (!proto) |
---|
| 3165 | + proto = po->num; |
---|
| 3166 | + |
---|
3149 | 3167 | rcu_read_lock(); |
---|
3150 | 3168 | |
---|
3151 | 3169 | if (po->fanout) { |
---|
.. | .. |
---|
3248 | 3266 | memcpy(name, uaddr->sa_data, sizeof(uaddr->sa_data)); |
---|
3249 | 3267 | name[sizeof(uaddr->sa_data)] = 0; |
---|
3250 | 3268 | |
---|
3251 | | - return packet_do_bind(sk, name, 0, pkt_sk(sk)->num); |
---|
| 3269 | + return packet_do_bind(sk, name, 0, 0); |
---|
3252 | 3270 | } |
---|
3253 | 3271 | |
---|
3254 | 3272 | static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) |
---|
.. | .. |
---|
3265 | 3283 | if (sll->sll_family != AF_PACKET) |
---|
3266 | 3284 | return -EINVAL; |
---|
3267 | 3285 | |
---|
3268 | | - return packet_do_bind(sk, NULL, sll->sll_ifindex, |
---|
3269 | | - sll->sll_protocol ? : pkt_sk(sk)->num); |
---|
| 3286 | + return packet_do_bind(sk, NULL, sll->sll_ifindex, sll->sll_protocol); |
---|
3270 | 3287 | } |
---|
3271 | 3288 | |
---|
3272 | 3289 | static struct proto packet_proto = { |
---|
.. | .. |
---|
3472 | 3489 | memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa, copy_len); |
---|
3473 | 3490 | } |
---|
3474 | 3491 | |
---|
3475 | | - if (pkt_sk(sk)->auxdata) { |
---|
| 3492 | + if (packet_sock_flag(pkt_sk(sk), PACKET_SOCK_AUXDATA)) { |
---|
3476 | 3493 | struct tpacket_auxdata aux; |
---|
3477 | 3494 | |
---|
3478 | 3495 | aux.tp_status = TP_STATUS_USER; |
---|
.. | .. |
---|
3856 | 3873 | if (copy_from_sockptr(&val, optval, sizeof(val))) |
---|
3857 | 3874 | return -EFAULT; |
---|
3858 | 3875 | |
---|
3859 | | - lock_sock(sk); |
---|
3860 | | - po->auxdata = !!val; |
---|
3861 | | - release_sock(sk); |
---|
| 3876 | + packet_sock_flag_set(po, PACKET_SOCK_AUXDATA, val); |
---|
3862 | 3877 | return 0; |
---|
3863 | 3878 | } |
---|
3864 | 3879 | case PACKET_ORIGDEV: |
---|
.. | .. |
---|
3870 | 3885 | if (copy_from_sockptr(&val, optval, sizeof(val))) |
---|
3871 | 3886 | return -EFAULT; |
---|
3872 | 3887 | |
---|
3873 | | - lock_sock(sk); |
---|
3874 | | - po->origdev = !!val; |
---|
3875 | | - release_sock(sk); |
---|
| 3888 | + packet_sock_flag_set(po, PACKET_SOCK_ORIGDEV, val); |
---|
3876 | 3889 | return 0; |
---|
3877 | 3890 | } |
---|
3878 | 3891 | case PACKET_VNET_HDR: |
---|
.. | .. |
---|
3969 | 3982 | if (copy_from_sockptr(&val, optval, sizeof(val))) |
---|
3970 | 3983 | return -EFAULT; |
---|
3971 | 3984 | |
---|
3972 | | - po->xmit = val ? packet_direct_xmit : dev_queue_xmit; |
---|
| 3985 | + /* Paired with all lockless reads of po->xmit */ |
---|
| 3986 | + WRITE_ONCE(po->xmit, val ? packet_direct_xmit : dev_queue_xmit); |
---|
3973 | 3987 | return 0; |
---|
3974 | 3988 | } |
---|
3975 | 3989 | default: |
---|
.. | .. |
---|
4020 | 4034 | |
---|
4021 | 4035 | break; |
---|
4022 | 4036 | case PACKET_AUXDATA: |
---|
4023 | | - val = po->auxdata; |
---|
| 4037 | + val = packet_sock_flag(po, PACKET_SOCK_AUXDATA); |
---|
4024 | 4038 | break; |
---|
4025 | 4039 | case PACKET_ORIGDEV: |
---|
4026 | | - val = po->origdev; |
---|
| 4040 | + val = packet_sock_flag(po, PACKET_SOCK_ORIGDEV); |
---|
4027 | 4041 | break; |
---|
4028 | 4042 | case PACKET_VNET_HDR: |
---|
4029 | 4043 | val = po->has_vnet_hdr; |
---|