.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
1 | 2 | /* |
---|
2 | 3 | * IPV6 GSO/GRO offload support |
---|
3 | 4 | * Linux INET implementation |
---|
4 | 5 | * |
---|
5 | 6 | * Copyright (C) 2016 secunet Security Networks AG |
---|
6 | 7 | * Author: Steffen Klassert <steffen.klassert@secunet.com> |
---|
7 | | - * |
---|
8 | | - * This program is free software; you can redistribute it and/or modify it |
---|
9 | | - * under the terms and conditions of the GNU General Public License, |
---|
10 | | - * version 2, as published by the Free Software Foundation. |
---|
11 | 8 | * |
---|
12 | 9 | * ESP GRO support |
---|
13 | 10 | */ |
---|
.. | .. |
---|
68 | 65 | |
---|
69 | 66 | xo = xfrm_offload(skb); |
---|
70 | 67 | if (!xo || !(xo->flags & CRYPTO_DONE)) { |
---|
71 | | - err = secpath_set(skb); |
---|
72 | | - if (err) |
---|
| 68 | + struct sec_path *sp = secpath_set(skb); |
---|
| 69 | + |
---|
| 70 | + if (!sp) |
---|
73 | 71 | goto out; |
---|
74 | 72 | |
---|
75 | | - if (skb->sp->len == XFRM_MAX_DEPTH) |
---|
76 | | - goto out; |
---|
| 73 | + if (sp->len == XFRM_MAX_DEPTH) |
---|
| 74 | + goto out_reset; |
---|
77 | 75 | |
---|
78 | 76 | x = xfrm_state_lookup(dev_net(skb->dev), skb->mark, |
---|
79 | 77 | (xfrm_address_t *)&ipv6_hdr(skb)->daddr, |
---|
80 | 78 | spi, IPPROTO_ESP, AF_INET6); |
---|
81 | 79 | if (!x) |
---|
82 | | - goto out; |
---|
| 80 | + goto out_reset; |
---|
83 | 81 | |
---|
84 | | - skb->sp->xvec[skb->sp->len++] = x; |
---|
85 | | - skb->sp->olen++; |
---|
| 82 | + skb->mark = xfrm_smark_get(skb->mark, x); |
---|
| 83 | + |
---|
| 84 | + sp->xvec[sp->len++] = x; |
---|
| 85 | + sp->olen++; |
---|
86 | 86 | |
---|
87 | 87 | xo = xfrm_offload(skb); |
---|
88 | | - if (!xo) { |
---|
89 | | - xfrm_state_put(x); |
---|
90 | | - goto out; |
---|
91 | | - } |
---|
| 88 | + if (!xo) |
---|
| 89 | + goto out_reset; |
---|
92 | 90 | } |
---|
93 | 91 | |
---|
94 | 92 | xo->flags |= XFRM_GRO; |
---|
.. | .. |
---|
108 | 106 | xfrm_input(skb, IPPROTO_ESP, spi, -2); |
---|
109 | 107 | |
---|
110 | 108 | return ERR_PTR(-EINPROGRESS); |
---|
| 109 | +out_reset: |
---|
| 110 | + secpath_reset(skb); |
---|
111 | 111 | out: |
---|
112 | 112 | skb_push(skb, offset); |
---|
113 | 113 | NAPI_GRO_CB(skb)->same_flow = 0; |
---|
.. | .. |
---|
125 | 125 | |
---|
126 | 126 | skb_push(skb, -skb_network_offset(skb)); |
---|
127 | 127 | |
---|
128 | | - if (x->outer_mode->encap == XFRM_MODE_TRANSPORT) { |
---|
| 128 | + if (x->outer_mode.encap == XFRM_MODE_TRANSPORT) { |
---|
129 | 129 | __be16 frag; |
---|
130 | 130 | |
---|
131 | 131 | ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &proto, &frag); |
---|
.. | .. |
---|
140 | 140 | xo->proto = proto; |
---|
141 | 141 | } |
---|
142 | 142 | |
---|
| 143 | +static struct sk_buff *xfrm6_tunnel_gso_segment(struct xfrm_state *x, |
---|
| 144 | + struct sk_buff *skb, |
---|
| 145 | + netdev_features_t features) |
---|
| 146 | +{ |
---|
| 147 | + __skb_push(skb, skb->mac_len); |
---|
| 148 | + return skb_mac_gso_segment(skb, features); |
---|
| 149 | +} |
---|
| 150 | + |
---|
| 151 | +static struct sk_buff *xfrm6_transport_gso_segment(struct xfrm_state *x, |
---|
| 152 | + struct sk_buff *skb, |
---|
| 153 | + netdev_features_t features) |
---|
| 154 | +{ |
---|
| 155 | + const struct net_offload *ops; |
---|
| 156 | + struct sk_buff *segs = ERR_PTR(-EINVAL); |
---|
| 157 | + struct xfrm_offload *xo = xfrm_offload(skb); |
---|
| 158 | + |
---|
| 159 | + skb->transport_header += x->props.header_len; |
---|
| 160 | + ops = rcu_dereference(inet6_offloads[xo->proto]); |
---|
| 161 | + if (likely(ops && ops->callbacks.gso_segment)) |
---|
| 162 | + segs = ops->callbacks.gso_segment(skb, features); |
---|
| 163 | + |
---|
| 164 | + return segs; |
---|
| 165 | +} |
---|
| 166 | + |
---|
| 167 | +static struct sk_buff *xfrm6_beet_gso_segment(struct xfrm_state *x, |
---|
| 168 | + struct sk_buff *skb, |
---|
| 169 | + netdev_features_t features) |
---|
| 170 | +{ |
---|
| 171 | + struct xfrm_offload *xo = xfrm_offload(skb); |
---|
| 172 | + struct sk_buff *segs = ERR_PTR(-EINVAL); |
---|
| 173 | + const struct net_offload *ops; |
---|
| 174 | + u8 proto = xo->proto; |
---|
| 175 | + |
---|
| 176 | + skb->transport_header += x->props.header_len; |
---|
| 177 | + |
---|
| 178 | + if (x->sel.family != AF_INET6) { |
---|
| 179 | + skb->transport_header -= |
---|
| 180 | + (sizeof(struct ipv6hdr) - sizeof(struct iphdr)); |
---|
| 181 | + |
---|
| 182 | + if (proto == IPPROTO_BEETPH) { |
---|
| 183 | + struct ip_beet_phdr *ph = |
---|
| 184 | + (struct ip_beet_phdr *)skb->data; |
---|
| 185 | + |
---|
| 186 | + skb->transport_header += ph->hdrlen * 8; |
---|
| 187 | + proto = ph->nexthdr; |
---|
| 188 | + } else { |
---|
| 189 | + skb->transport_header -= IPV4_BEET_PHMAXLEN; |
---|
| 190 | + } |
---|
| 191 | + |
---|
| 192 | + if (proto == IPPROTO_TCP) |
---|
| 193 | + skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6; |
---|
| 194 | + } else { |
---|
| 195 | + __be16 frag; |
---|
| 196 | + |
---|
| 197 | + skb->transport_header += |
---|
| 198 | + ipv6_skip_exthdr(skb, 0, &proto, &frag); |
---|
| 199 | + } |
---|
| 200 | + |
---|
| 201 | + if (proto == IPPROTO_IPIP) |
---|
| 202 | + skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6; |
---|
| 203 | + |
---|
| 204 | + __skb_pull(skb, skb_transport_offset(skb)); |
---|
| 205 | + ops = rcu_dereference(inet6_offloads[proto]); |
---|
| 206 | + if (likely(ops && ops->callbacks.gso_segment)) |
---|
| 207 | + segs = ops->callbacks.gso_segment(skb, features); |
---|
| 208 | + |
---|
| 209 | + return segs; |
---|
| 210 | +} |
---|
| 211 | + |
---|
| 212 | +static struct sk_buff *xfrm6_outer_mode_gso_segment(struct xfrm_state *x, |
---|
| 213 | + struct sk_buff *skb, |
---|
| 214 | + netdev_features_t features) |
---|
| 215 | +{ |
---|
| 216 | + switch (x->outer_mode.encap) { |
---|
| 217 | + case XFRM_MODE_TUNNEL: |
---|
| 218 | + return xfrm6_tunnel_gso_segment(x, skb, features); |
---|
| 219 | + case XFRM_MODE_TRANSPORT: |
---|
| 220 | + return xfrm6_transport_gso_segment(x, skb, features); |
---|
| 221 | + case XFRM_MODE_BEET: |
---|
| 222 | + return xfrm6_beet_gso_segment(x, skb, features); |
---|
| 223 | + } |
---|
| 224 | + |
---|
| 225 | + return ERR_PTR(-EOPNOTSUPP); |
---|
| 226 | +} |
---|
| 227 | + |
---|
143 | 228 | static struct sk_buff *esp6_gso_segment(struct sk_buff *skb, |
---|
144 | 229 | netdev_features_t features) |
---|
145 | 230 | { |
---|
.. | .. |
---|
148 | 233 | struct crypto_aead *aead; |
---|
149 | 234 | netdev_features_t esp_features = features; |
---|
150 | 235 | struct xfrm_offload *xo = xfrm_offload(skb); |
---|
| 236 | + struct sec_path *sp; |
---|
151 | 237 | |
---|
152 | 238 | if (!xo) |
---|
153 | 239 | return ERR_PTR(-EINVAL); |
---|
.. | .. |
---|
155 | 241 | if (!(skb_shinfo(skb)->gso_type & SKB_GSO_ESP)) |
---|
156 | 242 | return ERR_PTR(-EINVAL); |
---|
157 | 243 | |
---|
158 | | - x = skb->sp->xvec[skb->sp->len - 1]; |
---|
| 244 | + sp = skb_sec_path(skb); |
---|
| 245 | + x = sp->xvec[sp->len - 1]; |
---|
159 | 246 | aead = x->data; |
---|
160 | 247 | esph = ip_esp_hdr(skb); |
---|
161 | 248 | |
---|
.. | .. |
---|
170 | 257 | skb->encap_hdr_csum = 1; |
---|
171 | 258 | |
---|
172 | 259 | if (!(features & NETIF_F_HW_ESP) || x->xso.dev != skb->dev) |
---|
173 | | - esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK); |
---|
| 260 | + esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK | |
---|
| 261 | + NETIF_F_SCTP_CRC); |
---|
174 | 262 | else if (!(features & NETIF_F_HW_ESP_TX_CSUM)) |
---|
175 | | - esp_features = features & ~NETIF_F_CSUM_MASK; |
---|
| 263 | + esp_features = features & ~(NETIF_F_CSUM_MASK | |
---|
| 264 | + NETIF_F_SCTP_CRC); |
---|
176 | 265 | |
---|
177 | 266 | xo->flags |= XFRM_GSO_SEGMENT; |
---|
178 | 267 | |
---|
179 | | - return x->outer_mode->gso_segment(x, skb, esp_features); |
---|
| 268 | + return xfrm6_outer_mode_gso_segment(x, skb, esp_features); |
---|
180 | 269 | } |
---|
181 | 270 | |
---|
182 | 271 | static int esp6_input_tail(struct xfrm_state *x, struct sk_buff *skb) |
---|
.. | .. |
---|
200 | 289 | int alen; |
---|
201 | 290 | int blksize; |
---|
202 | 291 | struct xfrm_offload *xo; |
---|
203 | | - struct ip_esp_hdr *esph; |
---|
204 | 292 | struct crypto_aead *aead; |
---|
205 | 293 | struct esp_info esp; |
---|
206 | 294 | bool hw_offload = true; |
---|
.. | .. |
---|
241 | 329 | |
---|
242 | 330 | seq = xo->seq.low; |
---|
243 | 331 | |
---|
244 | | - esph = ip_esp_hdr(skb); |
---|
245 | | - esph->spi = x->id.spi; |
---|
| 332 | + esp.esph = ip_esp_hdr(skb); |
---|
| 333 | + esp.esph->spi = x->id.spi; |
---|
246 | 334 | |
---|
247 | 335 | skb_push(skb, -skb_network_offset(skb)); |
---|
248 | 336 | |
---|
249 | 337 | if (xo->flags & XFRM_GSO_SEGMENT) { |
---|
250 | | - esph->seq_no = htonl(seq); |
---|
| 338 | + esp.esph->seq_no = htonl(seq); |
---|
251 | 339 | |
---|
252 | 340 | if (!skb_is_gso(skb)) |
---|
253 | 341 | xo->seq.low++; |
---|
254 | 342 | else |
---|
255 | 343 | xo->seq.low += skb_shinfo(skb)->gso_segs; |
---|
256 | 344 | } |
---|
| 345 | + |
---|
| 346 | + if (xo->seq.low < seq) |
---|
| 347 | + xo->seq.hi++; |
---|
257 | 348 | |
---|
258 | 349 | esp.seqno = cpu_to_be64(xo->seq.low + ((u64)xo->seq.hi << 32)); |
---|
259 | 350 | |
---|
.. | .. |
---|
263 | 354 | |
---|
264 | 355 | ipv6_hdr(skb)->payload_len = htons(len); |
---|
265 | 356 | |
---|
266 | | - if (hw_offload) |
---|
| 357 | + if (hw_offload) { |
---|
| 358 | + if (!skb_ext_add(skb, SKB_EXT_SEC_PATH)) |
---|
| 359 | + return -ENOMEM; |
---|
| 360 | + |
---|
| 361 | + xo = xfrm_offload(skb); |
---|
| 362 | + if (!xo) |
---|
| 363 | + return -EINVAL; |
---|
| 364 | + |
---|
| 365 | + xo->flags |= XFRM_XMIT; |
---|
267 | 366 | return 0; |
---|
| 367 | + } |
---|
268 | 368 | |
---|
269 | 369 | err = esp6_output_tail(x, skb, &esp); |
---|
270 | 370 | if (err) |
---|
.. | .. |
---|
272 | 372 | |
---|
273 | 373 | secpath_reset(skb); |
---|
274 | 374 | |
---|
| 375 | + if (skb_needs_linearize(skb, skb->dev->features) && |
---|
| 376 | + __skb_linearize(skb)) |
---|
| 377 | + return -ENOMEM; |
---|
275 | 378 | return 0; |
---|
276 | 379 | } |
---|
277 | 380 | |
---|
.. | .. |
---|
303 | 406 | |
---|
304 | 407 | static void __exit esp6_offload_exit(void) |
---|
305 | 408 | { |
---|
306 | | - if (xfrm_unregister_type_offload(&esp6_type_offload, AF_INET6) < 0) |
---|
307 | | - pr_info("%s: can't remove xfrm type offload\n", __func__); |
---|
308 | | - |
---|
| 409 | + xfrm_unregister_type_offload(&esp6_type_offload, AF_INET6); |
---|
309 | 410 | inet6_del_offload(&esp6_offload, IPPROTO_ESP); |
---|
310 | 411 | } |
---|
311 | 412 | |
---|
.. | .. |
---|
314 | 415 | MODULE_LICENSE("GPL"); |
---|
315 | 416 | MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>"); |
---|
316 | 417 | MODULE_ALIAS_XFRM_OFFLOAD_TYPE(AF_INET6, XFRM_PROTO_ESP); |
---|
| 418 | +MODULE_DESCRIPTION("IPV6 GSO/GRO offload support"); |
---|