.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-or-later |
---|
1 | 2 | /* |
---|
2 | 3 | * xfrm_device.c - IPsec device offloading code. |
---|
3 | 4 | * |
---|
.. | .. |
---|
5 | 6 | * |
---|
6 | 7 | * Author: |
---|
7 | 8 | * Steffen Klassert <steffen.klassert@secunet.com> |
---|
8 | | - * |
---|
9 | | - * This program is free software; you can redistribute it and/or |
---|
10 | | - * modify it under the terms of the GNU General Public License |
---|
11 | | - * as published by the Free Software Foundation; either version |
---|
12 | | - * 2 of the License, or (at your option) any later version. |
---|
13 | 9 | */ |
---|
14 | 10 | |
---|
15 | 11 | #include <linux/errno.h> |
---|
.. | .. |
---|
23 | 19 | #include <linux/notifier.h> |
---|
24 | 20 | |
---|
25 | 21 | #ifdef CONFIG_XFRM_OFFLOAD |
---|
| 22 | +static void __xfrm_transport_prep(struct xfrm_state *x, struct sk_buff *skb, |
---|
| 23 | + unsigned int hsize) |
---|
| 24 | +{ |
---|
| 25 | + struct xfrm_offload *xo = xfrm_offload(skb); |
---|
| 26 | + |
---|
| 27 | + skb_reset_mac_len(skb); |
---|
| 28 | + if (xo->flags & XFRM_GSO_SEGMENT) |
---|
| 29 | + skb->transport_header -= x->props.header_len; |
---|
| 30 | + |
---|
| 31 | + pskb_pull(skb, skb_transport_offset(skb) + x->props.header_len); |
---|
| 32 | +} |
---|
| 33 | + |
---|
| 34 | +static void __xfrm_mode_tunnel_prep(struct xfrm_state *x, struct sk_buff *skb, |
---|
| 35 | + unsigned int hsize) |
---|
| 36 | + |
---|
| 37 | +{ |
---|
| 38 | + struct xfrm_offload *xo = xfrm_offload(skb); |
---|
| 39 | + |
---|
| 40 | + if (xo->flags & XFRM_GSO_SEGMENT) |
---|
| 41 | + skb->transport_header = skb->network_header + hsize; |
---|
| 42 | + |
---|
| 43 | + skb_reset_mac_len(skb); |
---|
| 44 | + pskb_pull(skb, skb->mac_len + x->props.header_len); |
---|
| 45 | +} |
---|
| 46 | + |
---|
| 47 | +static void __xfrm_mode_beet_prep(struct xfrm_state *x, struct sk_buff *skb, |
---|
| 48 | + unsigned int hsize) |
---|
| 49 | +{ |
---|
| 50 | + struct xfrm_offload *xo = xfrm_offload(skb); |
---|
| 51 | + int phlen = 0; |
---|
| 52 | + |
---|
| 53 | + if (xo->flags & XFRM_GSO_SEGMENT) |
---|
| 54 | + skb->transport_header = skb->network_header + hsize; |
---|
| 55 | + |
---|
| 56 | + skb_reset_mac_len(skb); |
---|
| 57 | + if (x->sel.family != AF_INET6) { |
---|
| 58 | + phlen = IPV4_BEET_PHMAXLEN; |
---|
| 59 | + if (x->outer_mode.family == AF_INET6) |
---|
| 60 | + phlen += sizeof(struct ipv6hdr) - sizeof(struct iphdr); |
---|
| 61 | + } |
---|
| 62 | + |
---|
| 63 | + pskb_pull(skb, skb->mac_len + hsize + (x->props.header_len - phlen)); |
---|
| 64 | +} |
---|
| 65 | + |
---|
| 66 | +/* Adjust pointers into the packet when IPsec is done at layer2 */ |
---|
| 67 | +static void xfrm_outer_mode_prep(struct xfrm_state *x, struct sk_buff *skb) |
---|
| 68 | +{ |
---|
| 69 | + switch (x->outer_mode.encap) { |
---|
| 70 | + case XFRM_MODE_TUNNEL: |
---|
| 71 | + if (x->outer_mode.family == AF_INET) |
---|
| 72 | + return __xfrm_mode_tunnel_prep(x, skb, |
---|
| 73 | + sizeof(struct iphdr)); |
---|
| 74 | + if (x->outer_mode.family == AF_INET6) |
---|
| 75 | + return __xfrm_mode_tunnel_prep(x, skb, |
---|
| 76 | + sizeof(struct ipv6hdr)); |
---|
| 77 | + break; |
---|
| 78 | + case XFRM_MODE_TRANSPORT: |
---|
| 79 | + if (x->outer_mode.family == AF_INET) |
---|
| 80 | + return __xfrm_transport_prep(x, skb, |
---|
| 81 | + sizeof(struct iphdr)); |
---|
| 82 | + if (x->outer_mode.family == AF_INET6) |
---|
| 83 | + return __xfrm_transport_prep(x, skb, |
---|
| 84 | + sizeof(struct ipv6hdr)); |
---|
| 85 | + break; |
---|
| 86 | + case XFRM_MODE_BEET: |
---|
| 87 | + if (x->outer_mode.family == AF_INET) |
---|
| 88 | + return __xfrm_mode_beet_prep(x, skb, |
---|
| 89 | + sizeof(struct iphdr)); |
---|
| 90 | + if (x->outer_mode.family == AF_INET6) |
---|
| 91 | + return __xfrm_mode_beet_prep(x, skb, |
---|
| 92 | + sizeof(struct ipv6hdr)); |
---|
| 93 | + break; |
---|
| 94 | + case XFRM_MODE_ROUTEOPTIMIZATION: |
---|
| 95 | + case XFRM_MODE_IN_TRIGGER: |
---|
| 96 | + break; |
---|
| 97 | + } |
---|
| 98 | +} |
---|
| 99 | + |
---|
| 100 | +static inline bool xmit_xfrm_check_overflow(struct sk_buff *skb) |
---|
| 101 | +{ |
---|
| 102 | + struct xfrm_offload *xo = xfrm_offload(skb); |
---|
| 103 | + __u32 seq = xo->seq.low; |
---|
| 104 | + |
---|
| 105 | + seq += skb_shinfo(skb)->gso_segs; |
---|
| 106 | + if (unlikely(seq < xo->seq.low)) |
---|
| 107 | + return true; |
---|
| 108 | + |
---|
| 109 | + return false; |
---|
| 110 | +} |
---|
| 111 | + |
---|
26 | 112 | struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again) |
---|
27 | 113 | { |
---|
28 | 114 | int err; |
---|
29 | 115 | unsigned long flags; |
---|
30 | 116 | struct xfrm_state *x; |
---|
31 | | - struct sk_buff *skb2; |
---|
32 | 117 | struct softnet_data *sd; |
---|
| 118 | + struct sk_buff *skb2, *nskb, *pskb = NULL; |
---|
33 | 119 | netdev_features_t esp_features = features; |
---|
34 | 120 | struct xfrm_offload *xo = xfrm_offload(skb); |
---|
| 121 | + struct net_device *dev = skb->dev; |
---|
| 122 | + struct sec_path *sp; |
---|
35 | 123 | |
---|
36 | 124 | if (!xo || (xo->flags & XFRM_XMIT)) |
---|
37 | 125 | return skb; |
---|
.. | .. |
---|
39 | 127 | if (!(features & NETIF_F_HW_ESP)) |
---|
40 | 128 | esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK); |
---|
41 | 129 | |
---|
42 | | - x = skb->sp->xvec[skb->sp->len - 1]; |
---|
| 130 | + sp = skb_sec_path(skb); |
---|
| 131 | + x = sp->xvec[sp->len - 1]; |
---|
43 | 132 | if (xo->flags & XFRM_GRO || x->xso.flags & XFRM_OFFLOAD_INBOUND) |
---|
| 133 | + return skb; |
---|
| 134 | + |
---|
| 135 | + /* This skb was already validated on the upper/virtual dev */ |
---|
| 136 | + if ((x->xso.dev != dev) && (x->xso.real_dev == dev)) |
---|
44 | 137 | return skb; |
---|
45 | 138 | |
---|
46 | 139 | local_irq_save(flags); |
---|
.. | .. |
---|
53 | 146 | return skb; |
---|
54 | 147 | } |
---|
55 | 148 | |
---|
56 | | - xo->flags |= XFRM_XMIT; |
---|
| 149 | + if (skb_is_gso(skb) && (unlikely(x->xso.dev != dev) || |
---|
| 150 | + unlikely(xmit_xfrm_check_overflow(skb)))) { |
---|
| 151 | + struct sk_buff *segs; |
---|
57 | 152 | |
---|
58 | | - if (skb_is_gso(skb)) { |
---|
59 | | - struct net_device *dev = skb->dev; |
---|
| 153 | + /* Packet got rerouted, fixup features and segment it. */ |
---|
| 154 | + esp_features = esp_features & ~(NETIF_F_HW_ESP | NETIF_F_GSO_ESP); |
---|
60 | 155 | |
---|
61 | | - if (unlikely(x->xso.dev != dev)) { |
---|
62 | | - struct sk_buff *segs; |
---|
63 | | - |
---|
64 | | - /* Packet got rerouted, fixup features and segment it. */ |
---|
65 | | - esp_features = esp_features & ~(NETIF_F_HW_ESP |
---|
66 | | - | NETIF_F_GSO_ESP); |
---|
67 | | - |
---|
68 | | - segs = skb_gso_segment(skb, esp_features); |
---|
69 | | - if (IS_ERR(segs)) { |
---|
70 | | - kfree_skb(skb); |
---|
71 | | - atomic_long_inc(&dev->tx_dropped); |
---|
72 | | - return NULL; |
---|
73 | | - } else { |
---|
74 | | - consume_skb(skb); |
---|
75 | | - skb = segs; |
---|
76 | | - } |
---|
| 156 | + segs = skb_gso_segment(skb, esp_features); |
---|
| 157 | + if (IS_ERR(segs)) { |
---|
| 158 | + kfree_skb(skb); |
---|
| 159 | + atomic_long_inc(&dev->tx_dropped); |
---|
| 160 | + return NULL; |
---|
| 161 | + } else { |
---|
| 162 | + consume_skb(skb); |
---|
| 163 | + skb = segs; |
---|
77 | 164 | } |
---|
78 | 165 | } |
---|
79 | 166 | |
---|
80 | 167 | if (!skb->next) { |
---|
81 | | - x->outer_mode->xmit(x, skb); |
---|
| 168 | + esp_features |= skb->dev->gso_partial_features; |
---|
| 169 | + xfrm_outer_mode_prep(x, skb); |
---|
82 | 170 | |
---|
83 | 171 | xo->flags |= XFRM_DEV_RESUME; |
---|
84 | 172 | |
---|
.. | .. |
---|
97 | 185 | return skb; |
---|
98 | 186 | } |
---|
99 | 187 | |
---|
100 | | - skb2 = skb; |
---|
101 | | - |
---|
102 | | - do { |
---|
103 | | - struct sk_buff *nskb = skb2->next; |
---|
104 | | - skb2->next = NULL; |
---|
| 188 | + skb_list_walk_safe(skb, skb2, nskb) { |
---|
| 189 | + esp_features |= skb->dev->gso_partial_features; |
---|
| 190 | + skb_mark_not_on_list(skb2); |
---|
105 | 191 | |
---|
106 | 192 | xo = xfrm_offload(skb2); |
---|
107 | 193 | xo->flags |= XFRM_DEV_RESUME; |
---|
108 | 194 | |
---|
109 | | - x->outer_mode->xmit(x, skb2); |
---|
| 195 | + xfrm_outer_mode_prep(x, skb2); |
---|
110 | 196 | |
---|
111 | 197 | err = x->type_offload->xmit(x, skb2, esp_features); |
---|
112 | 198 | if (!err) { |
---|
.. | .. |
---|
119 | 205 | } else { |
---|
120 | 206 | if (skb == skb2) |
---|
121 | 207 | skb = nskb; |
---|
| 208 | + else |
---|
| 209 | + pskb->next = nskb; |
---|
122 | 210 | |
---|
123 | | - if (!skb) |
---|
124 | | - return NULL; |
---|
125 | | - |
---|
126 | | - goto skip_push; |
---|
| 211 | + continue; |
---|
127 | 212 | } |
---|
128 | 213 | |
---|
129 | 214 | skb_push(skb2, skb2->data - skb_mac_header(skb2)); |
---|
130 | | - |
---|
131 | | -skip_push: |
---|
132 | | - skb2 = nskb; |
---|
133 | | - } while (skb2); |
---|
| 215 | + pskb = skb2; |
---|
| 216 | + } |
---|
134 | 217 | |
---|
135 | 218 | return skb; |
---|
136 | 219 | } |
---|
.. | .. |
---|
151 | 234 | |
---|
152 | 235 | /* We don't yet support UDP encapsulation and TFC padding. */ |
---|
153 | 236 | if (x->encap || x->tfcpad) |
---|
| 237 | + return -EINVAL; |
---|
| 238 | + |
---|
| 239 | + if (xuo->flags & ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND)) |
---|
154 | 240 | return -EINVAL; |
---|
155 | 241 | |
---|
156 | 242 | dev = dev_get_by_index(net, xuo->ifindex); |
---|
.. | .. |
---|
189 | 275 | } |
---|
190 | 276 | |
---|
191 | 277 | xso->dev = dev; |
---|
| 278 | + xso->real_dev = dev; |
---|
192 | 279 | xso->num_exthdrs = 1; |
---|
193 | | - xso->flags = xuo->flags; |
---|
| 280 | + /* Don't forward bit that is not implemented */ |
---|
| 281 | + xso->flags = xuo->flags & ~XFRM_OFFLOAD_IPV6; |
---|
194 | 282 | |
---|
195 | 283 | err = dev->xfrmdev_ops->xdo_dev_state_add(x); |
---|
196 | 284 | if (err) { |
---|
| 285 | + xso->num_exthdrs = 0; |
---|
| 286 | + xso->flags = 0; |
---|
197 | 287 | xso->dev = NULL; |
---|
| 288 | + xso->real_dev = NULL; |
---|
198 | 289 | dev_put(dev); |
---|
199 | | - return err; |
---|
| 290 | + |
---|
| 291 | + if (err != -EOPNOTSUPP) |
---|
| 292 | + return err; |
---|
200 | 293 | } |
---|
201 | 294 | |
---|
202 | 295 | return 0; |
---|
.. | .. |
---|
214 | 307 | return false; |
---|
215 | 308 | |
---|
216 | 309 | if ((!dev || (dev == xfrm_dst_path(dst)->dev)) && |
---|
217 | | - (!xdst->child->xfrm && x->type->get_mtu)) { |
---|
218 | | - mtu = x->type->get_mtu(x, xdst->child_mtu_cached); |
---|
219 | | - |
---|
| 310 | + (!xdst->child->xfrm)) { |
---|
| 311 | + mtu = xfrm_state_mtu(x, xdst->child_mtu_cached); |
---|
220 | 312 | if (skb->len <= mtu) |
---|
221 | 313 | goto ok; |
---|
222 | 314 | |
---|
.. | .. |
---|
243 | 335 | unsigned long flags; |
---|
244 | 336 | |
---|
245 | 337 | rcu_read_lock(); |
---|
246 | | - txq = netdev_pick_tx(dev, skb, NULL); |
---|
| 338 | + txq = netdev_core_pick_tx(dev, skb, NULL); |
---|
247 | 339 | |
---|
248 | 340 | HARD_TX_LOCK(dev, txq, smp_processor_id()); |
---|
249 | 341 | if (!netif_xmit_frozen_or_stopped(txq)) |
---|