add REDIRECT

hc

2024-10-09 05e59e5fb0064c97a1c10921ecd549f2d4a58565

 kernel/net/ipv6/ip6_input.c
..
..
@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-or-later
1
2
 /*
2
3
  *	IPv6 input
3
4
  *	Linux INET6 implementation
..
..
@@ -7,11 +8,6 @@
7
8
  *	Ian P. Morris		<I.P.Morris@soton.ac.uk>
8
9
  *
9
10
  *	Based in linux/net/ipv4/ip_input.c
10
- *
11
- *	This program is free software; you can redistribute it and/or
12
- *      modify it under the terms of the GNU General Public License
13
- *      as published by the Free Software Foundation; either version
14
- *      2 of the License, or (at your option) any later version.
15
11
  */
16
12
 /* Changes
17
13
  *
..
..
@@ -29,6 +25,7 @@
29
25
 #include <linux/icmpv6.h>
30
26
 #include <linux/mroute6.h>
31
27
 #include <linux/slab.h>
28
+#include <linux/indirect_call_wrapper.h>
32
29
 
33
30
 #include <linux/netfilter.h>
34
31
 #include <linux/netfilter_ipv6.h>
..
..
@@ -47,18 +44,25 @@
47
44
 #include <net/inet_ecn.h>
48
45
 #include <net/dst_metadata.h>
49
46
 
47
+void udp_v6_early_demux(struct sk_buff *);
48
+void tcp_v6_early_demux(struct sk_buff *);
50
49
 static void ip6_rcv_finish_core(struct net *net, struct sock *sk,
51
50
 				struct sk_buff *skb)
52
51
 {
53
-	void (*edemux)(struct sk_buff *skb);
54
-
55
-	if (net->ipv4.sysctl_ip_early_demux && !skb_dst(skb) && skb->sk == NULL) {
56
-		const struct inet6_protocol *ipprot;
57
-
58
-		ipprot = rcu_dereference(inet6_protos[ipv6_hdr(skb)->nexthdr]);
59
-		if (ipprot && (edemux = READ_ONCE(ipprot->early_demux)))
60
-			edemux(skb);
52
+	if (READ_ONCE(net->ipv4.sysctl_ip_early_demux) &&
53
+	    !skb_dst(skb) && !skb->sk) {
54
+		switch (ipv6_hdr(skb)->nexthdr) {
55
+		case IPPROTO_TCP:
56
+			if (READ_ONCE(net->ipv4.sysctl_tcp_early_demux))
57
+				tcp_v6_early_demux(skb);
58
+			break;
59
+		case IPPROTO_UDP:
60
+			if (READ_ONCE(net->ipv4.sysctl_udp_early_demux))
61
+				udp_v6_early_demux(skb);
62
+			break;
63
+		}
61
64
 	}
65
+
62
66
 	if (!skb_valid_dst(skb))
63
67
 		ip6_route_input(skb);
64
68
 }
..
..
@@ -86,11 +90,27 @@
86
90
 	}
87
91
 }
88
92
 
93
+static bool ip6_can_use_hint(const struct sk_buff *skb,
94
+			     const struct sk_buff *hint)
95
+{
96
+	return hint && !skb_dst(skb) &&
97
+	       ipv6_addr_equal(&ipv6_hdr(hint)->daddr, &ipv6_hdr(skb)->daddr);
98
+}
99
+
100
+static struct sk_buff *ip6_extract_route_hint(const struct net *net,
101
+					      struct sk_buff *skb)
102
+{
103
+	if (fib6_routes_require_src(net) || fib6_has_custom_rules(net))
104
+		return NULL;
105
+
106
+	return skb;
107
+}
108
+
89
109
 static void ip6_list_rcv_finish(struct net *net, struct sock *sk,
90
110
 				struct list_head *head)
91
111
 {
112
+	struct sk_buff *skb, *next, *hint = NULL;
92
113
 	struct dst_entry *curr_dst = NULL;
93
-	struct sk_buff *skb, *next;
94
114
 	struct list_head sublist;
95
115
 
96
116
 	INIT_LIST_HEAD(&sublist);
..
..
@@ -104,9 +124,15 @@
104
124
 		skb = l3mdev_ip6_rcv(skb);
105
125
 		if (!skb)
106
126
 			continue;
107
-		ip6_rcv_finish_core(net, sk, skb);
127
+
128
+		if (ip6_can_use_hint(skb, hint))
129
+			skb_dst_copy(skb, hint);
130
+		else
131
+			ip6_rcv_finish_core(net, sk, skb);
108
132
 		dst = skb_dst(skb);
109
133
 		if (curr_dst != dst) {
134
+			hint = ip6_extract_route_hint(net, skb);
135
+
110
136
 			/* dispatch old sublist */
111
137
 			if (!list_empty(&sublist))
112
138
 				ip6_sublist_rcv_finish(&sublist);
..
..
@@ -180,7 +206,8 @@
180
206
 	 */
181
207
 	if ((ipv6_addr_loopback(&hdr->saddr) ||
182
208
 	     ipv6_addr_loopback(&hdr->daddr)) &&
183
-	     !(dev->flags & IFF_LOOPBACK))
209
+	    !(dev->flags & IFF_LOOPBACK) &&
210
+	    !netif_is_l3_master(dev))
184
211
 		goto err;
185
212
 
186
213
 	/* RFC4291 Errata ID: 3480
..
..
@@ -252,7 +279,8 @@
252
279
 	rcu_read_unlock();
253
280
 
254
281
 	/* Must drop socket now because of tproxy. */
255
-	skb_orphan(skb);
282
+	if (!skb_sk_is_prefetched(skb))
283
+		skb_orphan(skb);
256
284
 
257
285
 	return skb;
258
286
 err:
..
..
@@ -314,34 +342,36 @@
314
342
 		list_add_tail(&skb->list, &sublist);
315
343
 	}
316
344
 	/* dispatch final sublist */
317
-	ip6_sublist_rcv(&sublist, curr_dev, curr_net);
345
+	if (!list_empty(&sublist))
346
+		ip6_sublist_rcv(&sublist, curr_dev, curr_net);
318
347
 }
348
+
349
+INDIRECT_CALLABLE_DECLARE(int udpv6_rcv(struct sk_buff *));
350
+INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *));
319
351
 
320
352
 /*
321
353
  *	Deliver the packet to the host
322
354
  */
323
-
324
-
325
-static int ip6_input_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
355
+void ip6_protocol_deliver_rcu(struct net *net, struct sk_buff *skb, int nexthdr,
356
+			      bool have_final)
326
357
 {
327
358
 	const struct inet6_protocol *ipprot;
328
359
 	struct inet6_dev *idev;
329
360
 	unsigned int nhoff;
330
-	int nexthdr;
331
361
 	bool raw;
332
-	bool have_final = false;
333
362
 
334
363
 	/*
335
364
 	 *	Parse extension headers
336
365
 	 */
337
366
 
338
-	rcu_read_lock();
339
367
 resubmit:
340
368
 	idev = ip6_dst_idev(skb_dst(skb));
341
-	if (!pskb_pull(skb, skb_transport_offset(skb)))
342
-		goto discard;
343
369
 	nhoff = IP6CB(skb)->nhoff;
344
-	nexthdr = skb_network_header(skb)[nhoff];
370
+	if (!have_final) {
371
+		if (!pskb_pull(skb, skb_transport_offset(skb)))
372
+			goto discard;
373
+		nexthdr = skb_network_header(skb)[nhoff];
374
+	}
345
375
 
346
376
 resubmit_final:
347
377
 	raw = raw6_local_deliver(skb, nexthdr);
..
..
@@ -360,6 +390,8 @@
360
390
 			}
361
391
 		} else if (ipprot->flags & INET6_PROTO_FINAL) {
362
392
 			const struct ipv6hdr *hdr;
393
+			int sdif = inet6_sdif(skb);
394
+			struct net_device *dev;
363
395
 
364
396
 			/* Only do this once for first final protocol */
365
397
 			have_final = true;
..
..
@@ -367,14 +399,24 @@
367
399
 			/* Free reference early: we don't need it any more,
368
400
 			   and it may hold ip_conntrack module loaded
369
401
 			   indefinitely. */
370
-			nf_reset(skb);
402
+			nf_reset_ct(skb);
371
403
 
372
404
 			skb_postpull_rcsum(skb, skb_network_header(skb),
373
405
 					   skb_network_header_len(skb));
374
406
 			hdr = ipv6_hdr(skb);
407
+
408
+			/* skb->dev passed may be master dev for vrfs. */
409
+			if (sdif) {
410
+				dev = dev_get_by_index_rcu(net, sdif);
411
+				if (!dev)
412
+					goto discard;
413
+			} else {
414
+				dev = skb->dev;
415
+			}
416
+
375
417
 			if (ipv6_addr_is_multicast(&hdr->daddr) &&
376
-			    !ipv6_chk_mcast_addr(skb->dev, &hdr->daddr,
377
-			    &hdr->saddr) &&
418
+			    !ipv6_chk_mcast_addr(dev, &hdr->daddr,
419
+						 &hdr->saddr) &&
378
420
 			    !ipv6_is_mld(skb, nexthdr, skb_network_header_len(skb)))
379
421
 				goto discard;
380
422
 		}
..
..
@@ -382,7 +424,8 @@
382
424
 		    !xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))
383
425
 			goto discard;
384
426
 
385
-		ret = ipprot->handler(skb);
427
+		ret = INDIRECT_CALL_2(ipprot->handler, tcp_v6_rcv, udpv6_rcv,
428
+				      skb);
386
429
 		if (ret > 0) {
387
430
 			if (ipprot->flags & INET6_PROTO_FINAL) {
388
431
 				/* Not an extension header, most likely UDP
..
..
@@ -412,13 +455,19 @@
412
455
 			consume_skb(skb);
413
456
 		}
414
457
 	}
415
-	rcu_read_unlock();
416
-	return 0;
458
+	return;
417
459
 
418
460
 discard:
419
461
 	__IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS);
420
-	rcu_read_unlock();
421
462
 	kfree_skb(skb);
463
+}
464
+
465
+static int ip6_input_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
466
+{
467
+	rcu_read_lock();
468
+	ip6_protocol_deliver_rcu(net, skb, 0, false);
469
+	rcu_read_unlock();
470
+
422
471
 	return 0;
423
472
 }
424
473
 
..
..
@@ -433,21 +482,38 @@
433
482
 
434
483
 int ip6_mc_input(struct sk_buff *skb)
435
484
 {
485
+	int sdif = inet6_sdif(skb);
436
486
 	const struct ipv6hdr *hdr;
487
+	struct net_device *dev;
437
488
 	bool deliver;
438
489
 
439
490
 	__IP6_UPD_PO_STATS(dev_net(skb_dst(skb)->dev),
440
491
 			 __in6_dev_get_safely(skb->dev), IPSTATS_MIB_INMCAST,
441
492
 			 skb->len);
442
493
 
494
+	/* skb->dev passed may be master dev for vrfs. */
495
+	if (sdif) {
496
+		rcu_read_lock();
497
+		dev = dev_get_by_index_rcu(dev_net(skb->dev), sdif);
498
+		if (!dev) {
499
+			rcu_read_unlock();
500
+			kfree_skb(skb);
501
+			return -ENODEV;
502
+		}
503
+	} else {
504
+		dev = skb->dev;
505
+	}
506
+
443
507
 	hdr = ipv6_hdr(skb);
444
-	deliver = ipv6_chk_mcast_addr(skb->dev, &hdr->daddr, NULL);
508
+	deliver = ipv6_chk_mcast_addr(dev, &hdr->daddr, NULL);
509
+	if (sdif)
510
+		rcu_read_unlock();
445
511
 
446
512
 #ifdef CONFIG_IPV6_MROUTE
447
513
 	/*
448
514
 	 *      IPv6 multicast router mode is now supported ;)
449
515
 	 */
450
-	if (dev_net(skb->dev)->ipv6.devconf_all->mc_forwarding &&
516
+	if (atomic_read(&dev_net(skb->dev)->ipv6.devconf_all->mc_forwarding) &&
451
517
 	    !(ipv6_addr_type(&hdr->daddr) &
452
518
 	      (IPV6_ADDR_LOOPBACK|IPV6_ADDR_LINKLOCAL)) &&
453
519
 	    likely(!(IP6CB(skb)->flags & IP6SKB_FORWARDED))) {