update kernel to 5.10.198

hc

2024-01-03 2f7c68cb55ecb7331f2381deb497c27155f32faf

 kernel/tools/testing/selftests/net/udpgso_bench_tx.c
..
..
@@ -5,6 +5,8 @@
5
5
 #include <arpa/inet.h>
6
6
 #include <errno.h>
7
7
 #include <error.h>
8
+#include <linux/errqueue.h>
9
+#include <linux/net_tstamp.h>
8
10
 #include <netinet/if_ether.h>
9
11
 #include <netinet/in.h>
10
12
 #include <netinet/ip.h>
..
..
@@ -19,8 +21,11 @@
19
21
 #include <string.h>
20
22
 #include <sys/socket.h>
21
23
 #include <sys/time.h>
24
+#include <sys/poll.h>
22
25
 #include <sys/types.h>
23
26
 #include <unistd.h>
27
+
28
+#include "../kselftest.h"
24
29
 
25
30
 #ifndef ETH_MAX_MTU
26
31
 #define ETH_MAX_MTU 0xFFFFU
..
..
@@ -34,8 +39,16 @@
34
39
 #define SO_ZEROCOPY	60
35
40
 #endif
36
41
 
42
+#ifndef SO_EE_ORIGIN_ZEROCOPY
43
+#define SO_EE_ORIGIN_ZEROCOPY 5
44
+#endif
45
+
37
46
 #ifndef MSG_ZEROCOPY
38
47
 #define MSG_ZEROCOPY	0x4000000
48
+#endif
49
+
50
+#ifndef ENOTSUPP
51
+#define ENOTSUPP	524
39
52
 #endif
40
53
 
41
54
 #define NUM_PKT		100
..
..
@@ -48,10 +61,25 @@
48
61
 static int	cfg_payload_len	= (1472 * 42);
49
62
 static int	cfg_port	= 8000;
50
63
 static int	cfg_runtime_ms	= -1;
64
+static bool	cfg_poll;
65
+static int	cfg_poll_loop_timeout_ms = 2000;
51
66
 static bool	cfg_segment;
52
67
 static bool	cfg_sendmmsg;
53
68
 static bool	cfg_tcp;
69
+static uint32_t	cfg_tx_ts = SOF_TIMESTAMPING_TX_SOFTWARE;
70
+static bool	cfg_tx_tstamp;
71
+static bool	cfg_audit;
72
+static bool	cfg_verbose;
54
73
 static bool	cfg_zerocopy;
74
+static int	cfg_msg_nr;
75
+static uint16_t	cfg_gso_size;
76
+static unsigned long total_num_msgs;
77
+static unsigned long total_num_sends;
78
+static unsigned long stat_tx_ts;
79
+static unsigned long stat_tx_ts_errors;
80
+static unsigned long tstart;
81
+static unsigned long tend;
82
+static unsigned long stat_zcopies;
55
83
 
56
84
 static socklen_t cfg_alen;
57
85
 static struct sockaddr_storage cfg_dst_addr;
..
..
@@ -108,21 +136,138 @@
108
136
 	}
109
137
 }
110
138
 
111
-static void flush_zerocopy(int fd)
139
+static void flush_cmsg(struct cmsghdr *cmsg)
112
140
 {
113
-	struct msghdr msg = {0};	/* flush */
141
+	struct sock_extended_err *err;
142
+	struct scm_timestamping *tss;
143
+	__u32 lo;
144
+	__u32 hi;
145
+	int i;
146
+
147
+	switch (cmsg->cmsg_level) {
148
+	case SOL_SOCKET:
149
+		if (cmsg->cmsg_type == SO_TIMESTAMPING) {
150
+			i = (cfg_tx_ts == SOF_TIMESTAMPING_TX_HARDWARE) ? 2 : 0;
151
+			tss = (struct scm_timestamping *)CMSG_DATA(cmsg);
152
+			if (tss->ts[i].tv_sec == 0)
153
+				stat_tx_ts_errors++;
154
+		} else {
155
+			error(1, 0, "unknown SOL_SOCKET cmsg type=%u\n",
156
+			      cmsg->cmsg_type);
157
+		}
158
+		break;
159
+	case SOL_IP:
160
+	case SOL_IPV6:
161
+		switch (cmsg->cmsg_type) {
162
+		case IP_RECVERR:
163
+		case IPV6_RECVERR:
164
+		{
165
+			err = (struct sock_extended_err *)CMSG_DATA(cmsg);
166
+			switch (err->ee_origin) {
167
+			case SO_EE_ORIGIN_TIMESTAMPING:
168
+				/* Got a TX timestamp from error queue */
169
+				stat_tx_ts++;
170
+				break;
171
+			case SO_EE_ORIGIN_ICMP:
172
+			case SO_EE_ORIGIN_ICMP6:
173
+				if (cfg_verbose)
174
+					fprintf(stderr,
175
+						"received ICMP error: type=%u, code=%u\n",
176
+						err->ee_type, err->ee_code);
177
+				break;
178
+			case SO_EE_ORIGIN_ZEROCOPY:
179
+			{
180
+				lo = err->ee_info;
181
+				hi = err->ee_data;
182
+				/* range of IDs acknowledged */
183
+				stat_zcopies += hi - lo + 1;
184
+				break;
185
+			}
186
+			case SO_EE_ORIGIN_LOCAL:
187
+				if (cfg_verbose)
188
+					fprintf(stderr,
189
+						"received packet with local origin: %u\n",
190
+						err->ee_origin);
191
+				break;
192
+			default:
193
+				error(0, 1, "received packet with origin: %u",
194
+				      err->ee_origin);
195
+			}
196
+			break;
197
+		}
198
+		default:
199
+			error(0, 1, "unknown IP msg type=%u\n",
200
+			      cmsg->cmsg_type);
201
+			break;
202
+		}
203
+		break;
204
+	default:
205
+		error(0, 1, "unknown cmsg level=%u\n",
206
+		      cmsg->cmsg_level);
207
+	}
208
+}
209
+
210
+static void flush_errqueue_recv(int fd)
211
+{
212
+	char control[CMSG_SPACE(sizeof(struct scm_timestamping)) +
213
+		     CMSG_SPACE(sizeof(struct sock_extended_err)) +
214
+		     CMSG_SPACE(sizeof(struct sockaddr_in6))] = {0};
215
+	struct msghdr msg = {0};
216
+	struct cmsghdr *cmsg;
114
217
 	int ret;
115
218
 
116
219
 	while (1) {
220
+		msg.msg_control = control;
221
+		msg.msg_controllen = sizeof(control);
117
222
 		ret = recvmsg(fd, &msg, MSG_ERRQUEUE);
118
223
 		if (ret == -1 && errno == EAGAIN)
119
224
 			break;
120
225
 		if (ret == -1)
121
226
 			error(1, errno, "errqueue");
122
-		if (msg.msg_flags != (MSG_ERRQUEUE | MSG_CTRUNC))
227
+		if (msg.msg_flags != MSG_ERRQUEUE)
123
228
 			error(1, 0, "errqueue: flags 0x%x\n", msg.msg_flags);
229
+		if (cfg_audit) {
230
+			for (cmsg = CMSG_FIRSTHDR(&msg);
231
+					cmsg;
232
+					cmsg = CMSG_NXTHDR(&msg, cmsg))
233
+				flush_cmsg(cmsg);
234
+		}
124
235
 		msg.msg_flags = 0;
125
236
 	}
237
+}
238
+
239
+static void flush_errqueue(int fd, const bool do_poll,
240
+			   unsigned long poll_timeout, const bool poll_err)
241
+{
242
+	if (do_poll) {
243
+		struct pollfd fds = {0};
244
+		int ret;
245
+
246
+		fds.fd = fd;
247
+		ret = poll(&fds, 1, poll_timeout);
248
+		if (ret == 0) {
249
+			if ((cfg_verbose) && (poll_err))
250
+				fprintf(stderr, "poll timeout\n");
251
+		} else if (ret < 0) {
252
+			error(1, errno, "poll");
253
+		}
254
+	}
255
+
256
+	flush_errqueue_recv(fd);
257
+}
258
+
259
+static void flush_errqueue_retry(int fd, unsigned long num_sends)
260
+{
261
+	unsigned long tnow, tstop;
262
+	bool first_try = true;
263
+
264
+	tnow = gettimeofday_ms();
265
+	tstop = tnow + cfg_poll_loop_timeout_ms;
266
+	do {
267
+		flush_errqueue(fd, true, tstop - tnow, first_try);
268
+		first_try = false;
269
+		tnow = gettimeofday_ms();
270
+	} while ((stat_zcopies != num_sends) && (tnow < tstop));
126
271
 }
127
272
 
128
273
 static int send_tcp(int fd, char *data)
..
..
@@ -166,15 +311,39 @@
166
311
 	return count;
167
312
 }
168
313
 
314
+static void send_ts_cmsg(struct cmsghdr *cm)
315
+{
316
+	uint32_t *valp;
317
+
318
+	cm->cmsg_level = SOL_SOCKET;
319
+	cm->cmsg_type = SO_TIMESTAMPING;
320
+	cm->cmsg_len = CMSG_LEN(sizeof(cfg_tx_ts));
321
+	valp = (void *)CMSG_DATA(cm);
322
+	*valp = cfg_tx_ts;
323
+}
324
+
169
325
 static int send_udp_sendmmsg(int fd, char *data)
170
326
 {
327
+	char control[CMSG_SPACE(sizeof(cfg_tx_ts))] = {0};
171
328
 	const int max_nr_msg = ETH_MAX_MTU / ETH_DATA_LEN;
172
329
 	struct mmsghdr mmsgs[max_nr_msg];
173
330
 	struct iovec iov[max_nr_msg];
174
331
 	unsigned int off = 0, left;
332
+	size_t msg_controllen = 0;
175
333
 	int i = 0, ret;
176
334
 
177
335
 	memset(mmsgs, 0, sizeof(mmsgs));
336
+
337
+	if (cfg_tx_tstamp) {
338
+		struct msghdr msg = {0};
339
+		struct cmsghdr *cmsg;
340
+
341
+		msg.msg_control = control;
342
+		msg.msg_controllen = sizeof(control);
343
+		cmsg = CMSG_FIRSTHDR(&msg);
344
+		send_ts_cmsg(cmsg);
345
+		msg_controllen += CMSG_SPACE(sizeof(cfg_tx_ts));
346
+	}
178
347
 
179
348
 	left = cfg_payload_len;
180
349
 	while (left) {
..
..
@@ -186,6 +355,13 @@
186
355
 
187
356
 		mmsgs[i].msg_hdr.msg_iov = iov + i;
188
357
 		mmsgs[i].msg_hdr.msg_iovlen = 1;
358
+
359
+		mmsgs[i].msg_hdr.msg_name = (void *)&cfg_dst_addr;
360
+		mmsgs[i].msg_hdr.msg_namelen = cfg_alen;
361
+		if (msg_controllen) {
362
+			mmsgs[i].msg_hdr.msg_control = control;
363
+			mmsgs[i].msg_hdr.msg_controllen = msg_controllen;
364
+		}
189
365
 
190
366
 		off += iov[i].iov_len;
191
367
 		left -= iov[i].iov_len;
..
..
@@ -205,16 +381,19 @@
205
381
 
206
382
 	cm->cmsg_level = SOL_UDP;
207
383
 	cm->cmsg_type = UDP_SEGMENT;
208
-	cm->cmsg_len = CMSG_LEN(sizeof(cfg_mss));
384
+	cm->cmsg_len = CMSG_LEN(sizeof(cfg_gso_size));
209
385
 	valp = (void *)CMSG_DATA(cm);
210
-	*valp = cfg_mss;
386
+	*valp = cfg_gso_size;
211
387
 }
212
388
 
213
389
 static int send_udp_segment(int fd, char *data)
214
390
 {
215
-	char control[CMSG_SPACE(sizeof(cfg_mss))] = {0};
391
+	char control[CMSG_SPACE(sizeof(cfg_gso_size)) +
392
+		     CMSG_SPACE(sizeof(cfg_tx_ts))] = {0};
216
393
 	struct msghdr msg = {0};
217
394
 	struct iovec iov = {0};
395
+	size_t msg_controllen;
396
+	struct cmsghdr *cmsg;
218
397
 	int ret;
219
398
 
220
399
 	iov.iov_base = data;
..
..
@@ -225,8 +404,16 @@
225
404
 
226
405
 	msg.msg_control = control;
227
406
 	msg.msg_controllen = sizeof(control);
228
-	send_udp_segment_cmsg(CMSG_FIRSTHDR(&msg));
407
+	cmsg = CMSG_FIRSTHDR(&msg);
408
+	send_udp_segment_cmsg(cmsg);
409
+	msg_controllen = CMSG_SPACE(sizeof(cfg_mss));
410
+	if (cfg_tx_tstamp) {
411
+		cmsg = CMSG_NXTHDR(&msg, cmsg);
412
+		send_ts_cmsg(cmsg);
413
+		msg_controllen += CMSG_SPACE(sizeof(cfg_tx_ts));
414
+	}
229
415
 
416
+	msg.msg_controllen = msg_controllen;
230
417
 	msg.msg_name = (void *)&cfg_dst_addr;
231
418
 	msg.msg_namelen = cfg_alen;
232
419
 
..
..
@@ -234,14 +421,16 @@
234
421
 	if (ret == -1)
235
422
 		error(1, errno, "sendmsg");
236
423
 	if (ret != iov.iov_len)
237
-		error(1, 0, "sendmsg: %u != %lu\n", ret, iov.iov_len);
424
+		error(1, 0, "sendmsg: %u != %llu\n", ret,
425
+			(unsigned long long)iov.iov_len);
238
426
 
239
427
 	return 1;
240
428
 }
241
429
 
242
430
 static void usage(const char *filepath)
243
431
 {
244
-	error(1, 0, "Usage: %s [-46cmStuz] [-C cpu] [-D dst ip] [-l secs] [-p port] [-s sendsize]",
432
+	error(1, 0, "Usage: %s [-46acmHPtTuvz] [-C cpu] [-D dst ip] [-l secs] "
433
+		    "[-L secs] [-M messagenr] [-p port] [-s sendsize] [-S gsosize]",
245
434
 		    filepath);
246
435
 }
247
436
 
..
..
@@ -251,7 +440,7 @@
251
440
 	int max_len, hdrlen;
252
441
 	int c;
253
442
 
254
-	while ((c = getopt(argc, argv, "46cC:D:l:mp:s:Stuz")) != -1) {
443
+	while ((c = getopt(argc, argv, "46acC:D:Hl:L:mM:p:s:PS:tTuvz")) != -1) {
255
444
 		switch (c) {
256
445
 		case '4':
257
446
 			if (cfg_family != PF_UNSPEC)
..
..
@@ -265,6 +454,9 @@
265
454
 			cfg_family = PF_INET6;
266
455
 			cfg_alen = sizeof(struct sockaddr_in6);
267
456
 			break;
457
+		case 'a':
458
+			cfg_audit = true;
459
+			break;
268
460
 		case 'c':
269
461
 			cfg_cache_trash = true;
270
462
 			break;
..
..
@@ -277,27 +469,49 @@
277
469
 		case 'l':
278
470
 			cfg_runtime_ms = strtoul(optarg, NULL, 10) * 1000;
279
471
 			break;
472
+		case 'L':
473
+			cfg_poll_loop_timeout_ms = strtoul(optarg, NULL, 10) * 1000;
474
+			break;
280
475
 		case 'm':
281
476
 			cfg_sendmmsg = true;
282
477
 			break;
478
+		case 'M':
479
+			cfg_msg_nr = strtoul(optarg, NULL, 10);
480
+			break;
283
481
 		case 'p':
284
482
 			cfg_port = strtoul(optarg, NULL, 0);
483
+			break;
484
+		case 'P':
485
+			cfg_poll = true;
285
486
 			break;
286
487
 		case 's':
287
488
 			cfg_payload_len = strtoul(optarg, NULL, 0);
288
489
 			break;
289
490
 		case 'S':
491
+			cfg_gso_size = strtoul(optarg, NULL, 0);
290
492
 			cfg_segment = true;
493
+			break;
494
+		case 'H':
495
+			cfg_tx_ts = SOF_TIMESTAMPING_TX_HARDWARE;
496
+			cfg_tx_tstamp = true;
291
497
 			break;
292
498
 		case 't':
293
499
 			cfg_tcp = true;
294
500
 			break;
501
+		case 'T':
502
+			cfg_tx_tstamp = true;
503
+			break;
295
504
 		case 'u':
296
505
 			cfg_connected = false;
506
+			break;
507
+		case 'v':
508
+			cfg_verbose = true;
297
509
 			break;
298
510
 		case 'z':
299
511
 			cfg_zerocopy = true;
300
512
 			break;
513
+		default:
514
+			exit(1);
301
515
 		}
302
516
 	}
303
517
 
..
..
@@ -315,6 +529,8 @@
315
529
 		error(1, 0, "connectionless tcp makes no sense");
316
530
 	if (cfg_segment && cfg_sendmmsg)
317
531
 		error(1, 0, "cannot combine segment offload and sendmmsg");
532
+	if (cfg_tx_tstamp && !(cfg_segment || cfg_sendmmsg))
533
+		error(1, 0, "Options -T and -H require either -S or -m option");
318
534
 
319
535
 	if (cfg_family == PF_INET)
320
536
 		hdrlen = sizeof(struct iphdr) + sizeof(struct udphdr);
..
..
@@ -323,6 +539,8 @@
323
539
 
324
540
 	cfg_mss = ETH_DATA_LEN - hdrlen;
325
541
 	max_len = ETH_MAX_MTU - hdrlen;
542
+	if (!cfg_gso_size)
543
+		cfg_gso_size = cfg_mss;
326
544
 
327
545
 	if (cfg_payload_len > max_len)
328
546
 		error(1, 0, "payload length %u exceeds max %u",
..
..
@@ -347,11 +565,80 @@
347
565
 		error(1, errno, "setsockopt path mtu");
348
566
 }
349
567
 
568
+static void set_tx_timestamping(int fd)
569
+{
570
+	int val = SOF_TIMESTAMPING_OPT_CMSG | SOF_TIMESTAMPING_OPT_ID |
571
+			SOF_TIMESTAMPING_OPT_TSONLY;
572
+
573
+	if (cfg_tx_ts == SOF_TIMESTAMPING_TX_SOFTWARE)
574
+		val |= SOF_TIMESTAMPING_SOFTWARE;
575
+	else
576
+		val |= SOF_TIMESTAMPING_RAW_HARDWARE;
577
+
578
+	if (setsockopt(fd, SOL_SOCKET, SO_TIMESTAMPING, &val, sizeof(val)))
579
+		error(1, errno, "setsockopt tx timestamping");
580
+}
581
+
582
+static void print_audit_report(unsigned long num_msgs, unsigned long num_sends)
583
+{
584
+	unsigned long tdelta;
585
+
586
+	tdelta = tend - tstart;
587
+	if (!tdelta)
588
+		return;
589
+
590
+	fprintf(stderr, "Summary over %lu.%03lu seconds...\n",
591
+			tdelta / 1000, tdelta % 1000);
592
+	fprintf(stderr,
593
+		"sum %s tx: %6lu MB/s %10lu calls (%lu/s) %10lu msgs (%lu/s)\n",
594
+		cfg_tcp ? "tcp" : "udp",
595
+		((num_msgs * cfg_payload_len) >> 10) / tdelta,
596
+		num_sends, num_sends * 1000 / tdelta,
597
+		num_msgs, num_msgs * 1000 / tdelta);
598
+
599
+	if (cfg_tx_tstamp) {
600
+		if (stat_tx_ts_errors)
601
+			error(1, 0,
602
+			      "Expected clean TX Timestamps: %9lu msgs received %6lu errors",
603
+			      stat_tx_ts, stat_tx_ts_errors);
604
+		if (stat_tx_ts != num_sends)
605
+			error(1, 0,
606
+			      "Unexpected number of TX Timestamps: %9lu expected %9lu received",
607
+			      num_sends, stat_tx_ts);
608
+		fprintf(stderr,
609
+			"Tx Timestamps: %19lu received %17lu errors\n",
610
+			stat_tx_ts, stat_tx_ts_errors);
611
+	}
612
+
613
+	if (cfg_zerocopy) {
614
+		if (stat_zcopies != num_sends)
615
+			error(1, 0, "Unexpected number of Zerocopy completions: %9lu expected %9lu received",
616
+			      num_sends, stat_zcopies);
617
+		fprintf(stderr,
618
+			"Zerocopy acks: %19lu\n",
619
+			stat_zcopies);
620
+	}
621
+}
622
+
623
+static void print_report(unsigned long num_msgs, unsigned long num_sends)
624
+{
625
+	fprintf(stderr,
626
+		"%s tx: %6lu MB/s %8lu calls/s %6lu msg/s\n",
627
+		cfg_tcp ? "tcp" : "udp",
628
+		(num_msgs * cfg_payload_len) >> 20,
629
+		num_sends, num_msgs);
630
+
631
+	if (cfg_audit) {
632
+		total_num_msgs += num_msgs;
633
+		total_num_sends += num_sends;
634
+	}
635
+}
636
+
350
637
 int main(int argc, char **argv)
351
638
 {
352
639
 	unsigned long num_msgs, num_sends;
353
640
 	unsigned long tnow, treport, tstop;
354
-	int fd, i, val;
641
+	int fd, i, val, ret;
355
642
 
356
643
 	parse_opts(argc, argv);
357
644
 
..
..
@@ -371,8 +658,16 @@
371
658
 
372
659
 	if (cfg_zerocopy) {
373
660
 		val = 1;
374
-		if (setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY, &val, sizeof(val)))
661
+
662
+		ret = setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY,
663
+				 &val, sizeof(val));
664
+		if (ret) {
665
+			if (errno == ENOPROTOOPT || errno == ENOTSUPP) {
666
+				fprintf(stderr, "SO_ZEROCOPY not supported");
667
+				exit(KSFT_SKIP);
668
+			}
375
669
 			error(1, errno, "setsockopt zerocopy");
670
+		}
376
671
 	}
377
672
 
378
673
 	if (cfg_connected &&
..
..
@@ -382,8 +677,13 @@
382
677
 	if (cfg_segment)
383
678
 		set_pmtu_discover(fd, cfg_family == PF_INET);
384
679
 
680
+	if (cfg_tx_tstamp)
681
+		set_tx_timestamping(fd);
682
+
385
683
 	num_msgs = num_sends = 0;
386
684
 	tnow = gettimeofday_ms();
685
+	tstart = tnow;
686
+	tend = tnow;
387
687
 	tstop = tnow + cfg_runtime_ms;
388
688
 	treport = tnow + 1000;
389
689
 
..
..
@@ -398,17 +698,15 @@
398
698
 		else
399
699
 			num_sends += send_udp(fd, buf[i]);
400
700
 		num_msgs++;
701
+		if ((cfg_zerocopy && ((num_msgs & 0xF) == 0)) || cfg_tx_tstamp)
702
+			flush_errqueue(fd, cfg_poll, 500, true);
401
703
 
402
-		if (cfg_zerocopy && ((num_msgs & 0xF) == 0))
403
-			flush_zerocopy(fd);
704
+		if (cfg_msg_nr && num_msgs >= cfg_msg_nr)
705
+			break;
404
706
 
405
707
 		tnow = gettimeofday_ms();
406
-		if (tnow > treport) {
407
-			fprintf(stderr,
408
-				"%s tx: %6lu MB/s %8lu calls/s %6lu msg/s\n",
409
-				cfg_tcp ? "tcp" : "udp",
410
-				(num_msgs * cfg_payload_len) >> 20,
411
-				num_sends, num_msgs);
708
+		if (tnow >= treport) {
709
+			print_report(num_msgs, num_sends);
412
710
 			num_msgs = num_sends = 0;
413
711
 			treport = tnow + 1000;
414
712
 		}
..
..
@@ -419,8 +717,18 @@
419
717
 
420
718
 	} while (!interrupted && (cfg_runtime_ms == -1 || tnow < tstop));
421
719
 
720
+	if (cfg_zerocopy || cfg_tx_tstamp)
721
+		flush_errqueue_retry(fd, num_sends);
722
+
422
723
 	if (close(fd))
423
724
 		error(1, errno, "close");
424
725
 
726
+	if (cfg_audit) {
727
+		tend = tnow;
728
+		total_num_msgs += num_msgs;
729
+		total_num_sends += num_sends;
730
+		print_audit_report(total_num_msgs, total_num_sends);
731
+	}
732
+
425
733
 	return 0;
426
734
 }