| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-or-later |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * IPV6 GSO/GRO offload support |
|---|
| 3 | 4 | * Linux INET6 implementation |
|---|
| 4 | | - * |
|---|
| 5 | | - * This program is free software; you can redistribute it and/or |
|---|
| 6 | | - * modify it under the terms of the GNU General Public License |
|---|
| 7 | | - * as published by the Free Software Foundation; either version |
|---|
| 8 | | - * 2 of the License, or (at your option) any later version. |
|---|
| 9 | 5 | * |
|---|
| 10 | 6 | * UDPv6 GSO support |
|---|
| 11 | 7 | */ |
|---|
| 12 | 8 | #include <linux/skbuff.h> |
|---|
| 13 | 9 | #include <linux/netdevice.h> |
|---|
| 10 | +#include <linux/indirect_call_wrapper.h> |
|---|
| 14 | 11 | #include <net/protocol.h> |
|---|
| 15 | 12 | #include <net/ipv6.h> |
|---|
| 16 | 13 | #include <net/udp.h> |
|---|
| .. | .. |
|---|
| 49 | 46 | goto out; |
|---|
| 50 | 47 | |
|---|
| 51 | 48 | if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) |
|---|
| 52 | | - return __udp_gso_segment(skb, features); |
|---|
| 49 | + return __udp_gso_segment(skb, features, true); |
|---|
| 53 | 50 | |
|---|
| 54 | 51 | /* Do software UFO. Complete and fill in the UDP checksum as HW cannot |
|---|
| 55 | 52 | * do checksum of UDP packets sent as multiple IP fragments. |
|---|
| .. | .. |
|---|
| 114 | 111 | return segs; |
|---|
| 115 | 112 | } |
|---|
| 116 | 113 | |
|---|
| 117 | | -static struct sk_buff *udp6_gro_receive(struct list_head *head, |
|---|
| 118 | | - struct sk_buff *skb) |
|---|
| 114 | +static struct sock *udp6_gro_lookup_skb(struct sk_buff *skb, __be16 sport, |
|---|
| 115 | + __be16 dport) |
|---|
| 116 | +{ |
|---|
| 117 | + const struct ipv6hdr *iph = skb_gro_network_header(skb); |
|---|
| 118 | + |
|---|
| 119 | + return __udp6_lib_lookup(dev_net(skb->dev), &iph->saddr, sport, |
|---|
| 120 | + &iph->daddr, dport, inet6_iif(skb), |
|---|
| 121 | + inet6_sdif(skb), &udp_table, NULL); |
|---|
| 122 | +} |
|---|
| 123 | + |
|---|
| 124 | +INDIRECT_CALLABLE_SCOPE |
|---|
| 125 | +struct sk_buff *udp6_gro_receive(struct list_head *head, struct sk_buff *skb) |
|---|
| 119 | 126 | { |
|---|
| 120 | 127 | struct udphdr *uh = udp_gro_udphdr(skb); |
|---|
| 128 | + struct sock *sk = NULL; |
|---|
| 129 | + struct sk_buff *pp; |
|---|
| 121 | 130 | |
|---|
| 122 | 131 | if (unlikely(!uh)) |
|---|
| 123 | 132 | goto flush; |
|---|
| .. | .. |
|---|
| 130 | 139 | ip6_gro_compute_pseudo)) |
|---|
| 131 | 140 | goto flush; |
|---|
| 132 | 141 | else if (uh->check) |
|---|
| 133 | | - skb_gro_checksum_try_convert(skb, IPPROTO_UDP, uh->check, |
|---|
| 142 | + skb_gro_checksum_try_convert(skb, IPPROTO_UDP, |
|---|
| 134 | 143 | ip6_gro_compute_pseudo); |
|---|
| 135 | 144 | |
|---|
| 136 | 145 | skip: |
|---|
| 137 | 146 | NAPI_GRO_CB(skb)->is_ipv6 = 1; |
|---|
| 138 | | - return udp_gro_receive(head, skb, uh, udp6_lib_lookup_skb); |
|---|
| 147 | + rcu_read_lock(); |
|---|
| 148 | + |
|---|
| 149 | + if (static_branch_unlikely(&udpv6_encap_needed_key)) |
|---|
| 150 | + sk = udp6_gro_lookup_skb(skb, uh->source, uh->dest); |
|---|
| 151 | + |
|---|
| 152 | + pp = udp_gro_receive(head, skb, uh, sk); |
|---|
| 153 | + rcu_read_unlock(); |
|---|
| 154 | + return pp; |
|---|
| 139 | 155 | |
|---|
| 140 | 156 | flush: |
|---|
| 141 | 157 | NAPI_GRO_CB(skb)->flush = 1; |
|---|
| 142 | 158 | return NULL; |
|---|
| 143 | 159 | } |
|---|
| 144 | 160 | |
|---|
| 145 | | -static int udp6_gro_complete(struct sk_buff *skb, int nhoff) |
|---|
| 161 | +INDIRECT_CALLABLE_SCOPE int udp6_gro_complete(struct sk_buff *skb, int nhoff) |
|---|
| 146 | 162 | { |
|---|
| 147 | 163 | const struct ipv6hdr *ipv6h = ipv6_hdr(skb); |
|---|
| 148 | 164 | struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); |
|---|
| 149 | 165 | |
|---|
| 150 | | - if (uh->check) { |
|---|
| 151 | | - skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; |
|---|
| 166 | + if (NAPI_GRO_CB(skb)->is_flist) { |
|---|
| 167 | + uh->len = htons(skb->len - nhoff); |
|---|
| 168 | + |
|---|
| 169 | + skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4); |
|---|
| 170 | + skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count; |
|---|
| 171 | + |
|---|
| 172 | + if (skb->ip_summed == CHECKSUM_UNNECESSARY) { |
|---|
| 173 | + if (skb->csum_level < SKB_MAX_CSUM_LEVEL) |
|---|
| 174 | + skb->csum_level++; |
|---|
| 175 | + } else { |
|---|
| 176 | + skb->ip_summed = CHECKSUM_UNNECESSARY; |
|---|
| 177 | + skb->csum_level = 0; |
|---|
| 178 | + } |
|---|
| 179 | + |
|---|
| 180 | + return 0; |
|---|
| 181 | + } |
|---|
| 182 | + |
|---|
| 183 | + if (uh->check) |
|---|
| 152 | 184 | uh->check = ~udp_v6_check(skb->len - nhoff, &ipv6h->saddr, |
|---|
| 153 | 185 | &ipv6h->daddr, 0); |
|---|
| 154 | | - } else { |
|---|
| 155 | | - skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; |
|---|
| 156 | | - } |
|---|
| 157 | 186 | |
|---|
| 158 | 187 | return udp_gro_complete(skb, nhoff, udp6_lib_lookup_skb); |
|---|
| 159 | 188 | } |
|---|