write in 30M

hc

2024-02-20 ea08eeccae9297f7aabd2ef7f0c2517ac4549acc

 kernel/net/ipv4/udp_offload.c
..
..
@@ -1,11 +1,7 @@
1
+// SPDX-License-Identifier: GPL-2.0-or-later
1
2
 /*
2
3
  *	IPV4 GSO/GRO offload support
3
4
  *	Linux INET implementation
4
- *
5
- *	This program is free software; you can redistribute it and/or
6
- *	modify it under the terms of the GNU General Public License
7
- *	as published by the Free Software Foundation; either version
8
- *	2 of the License, or (at your option) any later version.
9
5
  *
10
6
  *	UDPv4 GSO support
11
7
  */
..
..
@@ -13,6 +9,7 @@
13
9
 #include <linux/skbuff.h>
14
10
 #include <net/udp.h>
15
11
 #include <net/protocol.h>
12
+#include <net/inet_common.h>
16
13
 
17
14
 static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,
18
15
 	netdev_features_t features,
..
..
@@ -187,8 +184,81 @@
187
184
 }
188
185
 EXPORT_SYMBOL(skb_udp_tunnel_segment);
189
186
 
187
+static void __udpv4_gso_segment_csum(struct sk_buff *seg,
188
+				     __be32 *oldip, __be32 *newip,
189
+				     __be16 *oldport, __be16 *newport)
190
+{
191
+	struct udphdr *uh;
192
+	struct iphdr *iph;
193
+
194
+	if (*oldip == *newip && *oldport == *newport)
195
+		return;
196
+
197
+	uh = udp_hdr(seg);
198
+	iph = ip_hdr(seg);
199
+
200
+	if (uh->check) {
201
+		inet_proto_csum_replace4(&uh->check, seg, *oldip, *newip,
202
+					 true);
203
+		inet_proto_csum_replace2(&uh->check, seg, *oldport, *newport,
204
+					 false);
205
+		if (!uh->check)
206
+			uh->check = CSUM_MANGLED_0;
207
+	}
208
+	*oldport = *newport;
209
+
210
+	csum_replace4(&iph->check, *oldip, *newip);
211
+	*oldip = *newip;
212
+}
213
+
214
+static struct sk_buff *__udpv4_gso_segment_list_csum(struct sk_buff *segs)
215
+{
216
+	struct sk_buff *seg;
217
+	struct udphdr *uh, *uh2;
218
+	struct iphdr *iph, *iph2;
219
+
220
+	seg = segs;
221
+	uh = udp_hdr(seg);
222
+	iph = ip_hdr(seg);
223
+
224
+	if ((udp_hdr(seg)->dest == udp_hdr(seg->next)->dest) &&
225
+	    (udp_hdr(seg)->source == udp_hdr(seg->next)->source) &&
226
+	    (ip_hdr(seg)->daddr == ip_hdr(seg->next)->daddr) &&
227
+	    (ip_hdr(seg)->saddr == ip_hdr(seg->next)->saddr))
228
+		return segs;
229
+
230
+	while ((seg = seg->next)) {
231
+		uh2 = udp_hdr(seg);
232
+		iph2 = ip_hdr(seg);
233
+
234
+		__udpv4_gso_segment_csum(seg,
235
+					 &iph2->saddr, &iph->saddr,
236
+					 &uh2->source, &uh->source);
237
+		__udpv4_gso_segment_csum(seg,
238
+					 &iph2->daddr, &iph->daddr,
239
+					 &uh2->dest, &uh->dest);
240
+	}
241
+
242
+	return segs;
243
+}
244
+
245
+static struct sk_buff *__udp_gso_segment_list(struct sk_buff *skb,
246
+					      netdev_features_t features,
247
+					      bool is_ipv6)
248
+{
249
+	unsigned int mss = skb_shinfo(skb)->gso_size;
250
+
251
+	skb = skb_segment_list(skb, features, skb_mac_header_len(skb));
252
+	if (IS_ERR(skb))
253
+		return skb;
254
+
255
+	udp_hdr(skb)->len = htons(sizeof(struct udphdr) + mss);
256
+
257
+	return is_ipv6 ? skb : __udpv4_gso_segment_list_csum(skb);
258
+}
259
+
190
260
 struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
191
-				  netdev_features_t features)
261
+				  netdev_features_t features, bool is_ipv6)
192
262
 {
193
263
 	struct sock *sk = gso_skb->sk;
194
264
 	unsigned int sum_truesize = 0;
..
..
@@ -198,6 +268,9 @@
198
268
 	bool copy_dtor;
199
269
 	__sum16 check;
200
270
 	__be16 newlen;
271
+
272
+	if (skb_shinfo(gso_skb)->gso_type & SKB_GSO_FRAGLIST)
273
+		return __udp_gso_segment_list(gso_skb, features, is_ipv6);
201
274
 
202
275
 	mss = skb_shinfo(gso_skb)->gso_size;
203
276
 	if (gso_skb->len <= sizeof(*uh) + mss)
..
..
@@ -211,7 +284,7 @@
211
284
 		gso_skb->destructor = NULL;
212
285
 
213
286
 	segs = skb_segment(gso_skb, features);
214
-	if (unlikely(IS_ERR_OR_NULL(segs))) {
287
+	if (IS_ERR_OR_NULL(segs)) {
215
288
 		if (copy_dtor)
216
289
 			gso_skb->destructor = sock_wfree;
217
290
 		return segs;
..
..
@@ -311,7 +384,7 @@
311
384
 		goto out;
312
385
 
313
386
 	if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)
314
-		return __udp_gso_segment(skb, features);
387
+		return __udp_gso_segment(skb, features, false);
315
388
 
316
389
 	mss = skb_shinfo(skb)->gso_size;
317
390
 	if (unlikely(skb->len <= mss))
..
..
@@ -348,33 +421,116 @@
348
421
 	return segs;
349
422
 }
350
423
 
424
+#define UDP_GRO_CNT_MAX 64
425
+static struct sk_buff *udp_gro_receive_segment(struct list_head *head,
426
+					       struct sk_buff *skb)
427
+{
428
+	struct udphdr *uh = udp_gro_udphdr(skb);
429
+	struct sk_buff *pp = NULL;
430
+	struct udphdr *uh2;
431
+	struct sk_buff *p;
432
+	unsigned int ulen;
433
+	int ret = 0;
434
+
435
+	/* requires non zero csum, for symmetry with GSO */
436
+	if (!uh->check) {
437
+		NAPI_GRO_CB(skb)->flush = 1;
438
+		return NULL;
439
+	}
440
+
441
+	/* Do not deal with padded or malicious packets, sorry ! */
442
+	ulen = ntohs(uh->len);
443
+	if (ulen <= sizeof(*uh) || ulen != skb_gro_len(skb)) {
444
+		NAPI_GRO_CB(skb)->flush = 1;
445
+		return NULL;
446
+	}
447
+	/* pull encapsulating udp header */
448
+	skb_gro_pull(skb, sizeof(struct udphdr));
449
+
450
+	list_for_each_entry(p, head, list) {
451
+		if (!NAPI_GRO_CB(p)->same_flow)
452
+			continue;
453
+
454
+		uh2 = udp_hdr(p);
455
+
456
+		/* Match ports only, as csum is always non zero */
457
+		if ((*(u32 *)&uh->source != *(u32 *)&uh2->source)) {
458
+			NAPI_GRO_CB(p)->same_flow = 0;
459
+			continue;
460
+		}
461
+
462
+		if (NAPI_GRO_CB(skb)->is_flist != NAPI_GRO_CB(p)->is_flist) {
463
+			NAPI_GRO_CB(skb)->flush = 1;
464
+			return p;
465
+		}
466
+
467
+		/* Terminate the flow on len mismatch or if it grow "too much".
468
+		 * Under small packet flood GRO count could elsewhere grow a lot
469
+		 * leading to excessive truesize values.
470
+		 * On len mismatch merge the first packet shorter than gso_size,
471
+		 * otherwise complete the GRO packet.
472
+		 */
473
+		if (ulen > ntohs(uh2->len)) {
474
+			pp = p;
475
+		} else {
476
+			if (NAPI_GRO_CB(skb)->is_flist) {
477
+				if (!pskb_may_pull(skb, skb_gro_offset(skb))) {
478
+					NAPI_GRO_CB(skb)->flush = 1;
479
+					return NULL;
480
+				}
481
+				if ((skb->ip_summed != p->ip_summed) ||
482
+				    (skb->csum_level != p->csum_level)) {
483
+					NAPI_GRO_CB(skb)->flush = 1;
484
+					return NULL;
485
+				}
486
+				ret = skb_gro_receive_list(p, skb);
487
+			} else {
488
+				skb_gro_postpull_rcsum(skb, uh,
489
+						       sizeof(struct udphdr));
490
+
491
+				ret = skb_gro_receive(p, skb);
492
+			}
493
+		}
494
+
495
+		if (ret || ulen != ntohs(uh2->len) ||
496
+		    NAPI_GRO_CB(p)->count >= UDP_GRO_CNT_MAX)
497
+			pp = p;
498
+
499
+		return pp;
500
+	}
501
+
502
+	/* mismatch, but we never need to flush */
503
+	return NULL;
504
+}
505
+
351
506
 struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
352
-				struct udphdr *uh, udp_lookup_t lookup)
507
+				struct udphdr *uh, struct sock *sk)
353
508
 {
354
509
 	struct sk_buff *pp = NULL;
355
510
 	struct sk_buff *p;
356
511
 	struct udphdr *uh2;
357
512
 	unsigned int off = skb_gro_offset(skb);
358
513
 	int flush = 1;
359
-	struct sock *sk;
360
514
 
361
-	if (NAPI_GRO_CB(skb)->encap_mark ||
515
+	NAPI_GRO_CB(skb)->is_flist = 0;
516
+	if (skb->dev->features & NETIF_F_GRO_FRAGLIST)
517
+		NAPI_GRO_CB(skb)->is_flist = sk ? !udp_sk(sk)->gro_enabled: 1;
518
+
519
+	if ((sk && udp_sk(sk)->gro_enabled) || NAPI_GRO_CB(skb)->is_flist) {
520
+		pp = call_gro_receive(udp_gro_receive_segment, head, skb);
521
+		return pp;
522
+	}
523
+
524
+	if (!sk || NAPI_GRO_CB(skb)->encap_mark ||
362
525
 	    (uh->check && skb->ip_summed != CHECKSUM_PARTIAL &&
363
526
 	     NAPI_GRO_CB(skb)->csum_cnt == 0 &&
364
-	     !NAPI_GRO_CB(skb)->csum_valid))
527
+	     !NAPI_GRO_CB(skb)->csum_valid) ||
528
+	    !udp_sk(sk)->gro_receive)
365
529
 		goto out;
366
530
 
367
531
 	/* mark that this skb passed once through the tunnel gro layer */
368
532
 	NAPI_GRO_CB(skb)->encap_mark = 1;
369
533
 
370
-	rcu_read_lock();
371
-	sk = (*lookup)(skb, uh->source, uh->dest);
372
-
373
-	if (sk && udp_sk(sk)->gro_receive)
374
-		goto unflush;
375
-	goto out_unlock;
376
-
377
-unflush:
378
534
 	flush = 0;
379
535
 
380
536
 	list_for_each_entry(p, head, list) {
..
..
@@ -397,18 +553,28 @@
397
553
 	skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));
398
554
 	pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
399
555
 
400
-out_unlock:
401
-	rcu_read_unlock();
402
556
 out:
403
557
 	skb_gro_flush_final(skb, pp, flush);
404
558
 	return pp;
405
559
 }
406
560
 EXPORT_SYMBOL(udp_gro_receive);
407
561
 
408
-static struct sk_buff *udp4_gro_receive(struct list_head *head,
409
-					struct sk_buff *skb)
562
+static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport,
563
+					__be16 dport)
564
+{
565
+	const struct iphdr *iph = skb_gro_network_header(skb);
566
+
567
+	return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
568
+				 iph->daddr, dport, inet_iif(skb),
569
+				 inet_sdif(skb), &udp_table, NULL);
570
+}
571
+
572
+INDIRECT_CALLABLE_SCOPE
573
+struct sk_buff *udp4_gro_receive(struct list_head *head, struct sk_buff *skb)
410
574
 {
411
575
 	struct udphdr *uh = udp_gro_udphdr(skb);
576
+	struct sock *sk = NULL;
577
+	struct sk_buff *pp;
412
578
 
413
579
 	if (unlikely(!uh))
414
580
 		goto flush;
..
..
@@ -421,15 +587,39 @@
421
587
 						 inet_gro_compute_pseudo))
422
588
 		goto flush;
423
589
 	else if (uh->check)
424
-		skb_gro_checksum_try_convert(skb, IPPROTO_UDP, uh->check,
590
+		skb_gro_checksum_try_convert(skb, IPPROTO_UDP,
425
591
 					     inet_gro_compute_pseudo);
426
592
 skip:
427
593
 	NAPI_GRO_CB(skb)->is_ipv6 = 0;
428
-	return udp_gro_receive(head, skb, uh, udp4_lib_lookup_skb);
594
+	rcu_read_lock();
595
+
596
+	if (static_branch_unlikely(&udp_encap_needed_key))
597
+		sk = udp4_gro_lookup_skb(skb, uh->source, uh->dest);
598
+
599
+	pp = udp_gro_receive(head, skb, uh, sk);
600
+	rcu_read_unlock();
601
+	return pp;
429
602
 
430
603
 flush:
431
604
 	NAPI_GRO_CB(skb)->flush = 1;
432
605
 	return NULL;
606
+}
607
+
608
+static int udp_gro_complete_segment(struct sk_buff *skb)
609
+{
610
+	struct udphdr *uh = udp_hdr(skb);
611
+
612
+	skb->csum_start = (unsigned char *)uh - skb->head;
613
+	skb->csum_offset = offsetof(struct udphdr, check);
614
+	skb->ip_summed = CHECKSUM_PARTIAL;
615
+
616
+	skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
617
+	skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_L4;
618
+
619
+	if (skb->encapsulation)
620
+		skb->inner_transport_header = skb->transport_header;
621
+
622
+	return 0;
433
623
 }
434
624
 
435
625
 int udp_gro_complete(struct sk_buff *skb, int nhoff,
..
..
@@ -442,16 +632,22 @@
442
632
 
443
633
 	uh->len = newlen;
444
634
 
445
-	/* Set encapsulation before calling into inner gro_complete() functions
446
-	 * to make them set up the inner offsets.
447
-	 */
448
-	skb->encapsulation = 1;
449
-
450
635
 	rcu_read_lock();
451
-	sk = (*lookup)(skb, uh->source, uh->dest);
452
-	if (sk && udp_sk(sk)->gro_complete)
636
+	sk = INDIRECT_CALL_INET(lookup, udp6_lib_lookup_skb,
637
+				udp4_lib_lookup_skb, skb, uh->source, uh->dest);
638
+	if (sk && udp_sk(sk)->gro_complete) {
639
+		skb_shinfo(skb)->gso_type = uh->check ? SKB_GSO_UDP_TUNNEL_CSUM
640
+					: SKB_GSO_UDP_TUNNEL;
641
+
642
+		/* Set encapsulation before calling into inner gro_complete()
643
+		 * functions to make them set up the inner offsets.
644
+		 */
645
+		skb->encapsulation = 1;
453
646
 		err = udp_sk(sk)->gro_complete(sk, skb,
454
647
 				nhoff + sizeof(struct udphdr));
648
+	} else {
649
+		err = udp_gro_complete_segment(skb);
650
+	}
455
651
 	rcu_read_unlock();
456
652
 
457
653
 	if (skb->remcsum_offload)
..
..
@@ -461,18 +657,31 @@
461
657
 }
462
658
 EXPORT_SYMBOL(udp_gro_complete);
463
659
 
464
-static int udp4_gro_complete(struct sk_buff *skb, int nhoff)
660
+INDIRECT_CALLABLE_SCOPE int udp4_gro_complete(struct sk_buff *skb, int nhoff)
465
661
 {
466
662
 	const struct iphdr *iph = ip_hdr(skb);
467
663
 	struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
468
664
 
469
-	if (uh->check) {
470
-		skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
665
+	if (NAPI_GRO_CB(skb)->is_flist) {
666
+		uh->len = htons(skb->len - nhoff);
667
+
668
+		skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4);
669
+		skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
670
+
671
+		if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
672
+			if (skb->csum_level < SKB_MAX_CSUM_LEVEL)
673
+				skb->csum_level++;
674
+		} else {
675
+			skb->ip_summed = CHECKSUM_UNNECESSARY;
676
+			skb->csum_level = 0;
677
+		}
678
+
679
+		return 0;
680
+	}
681
+
682
+	if (uh->check)
471
683
 		uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr,
472
684
 					  iph->daddr, 0);
473
-	} else {
474
-		skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
475
-	}
476
685
 
477
686
 	return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);
478
687
 }